From f644e839350cda08d85ab90945c9ee75b8e3a5d1 Mon Sep 17 00:00:00 2001 From: Apoorv Parle <19315187+apparle@users.noreply.github.com> Date: Sat, 2 Nov 2024 14:11:17 -0700 Subject: [PATCH 1/6] Add ability to specify additional docker network and update documentation for it. Signed-off-by: Apoorv Parle <19315187+apparle@users.noreply.github.com> --- Containers/mastercontainer/start.sh | 8 +++ compose.yaml | 1 + php/src/Data/ConfigurationManager.php | 7 +++ php/src/Docker/DockerActionManager.php | 70 +++++++++++++++++--------- reverse-proxy.md | 7 +-- 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/Containers/mastercontainer/start.sh b/Containers/mastercontainer/start.sh index f87527cbb21..7dc94809abd 100644 --- a/Containers/mastercontainer/start.sh +++ b/Containers/mastercontainer/start.sh @@ -193,6 +193,14 @@ It is set to '$APACHE_IP_BINDING'." exit 1 fi fi +if [ -n "$APACHE_ADDITIONAL_NETWORK" ]; then + if ! echo "$APACHE_ADDITIONAL_NETWORK" | grep -q "^[a-zA-Z0-9_-]\+$"; then + print_red "You've set APACHE_ADDITIONAL_NETWORK but not to an allowed value. +It needs to be a string with letters, numbers, hyphens and underscores. +It is set to '$APACHE_ADDITIONAL_NETWORK'." + exit 1 + fi +fi if [ -n "$TALK_PORT" ]; then if ! check_if_number "$TALK_PORT"; then print_red "You provided an Talk port but did not only use numbers. diff --git a/compose.yaml b/compose.yaml index ecf4d588d86..438bba6ae14 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,7 @@ services: # AIO_COMMUNITY_CONTAINERS: # With this variable, you can add community containers very easily. See https://github.com/nextcloud/all-in-one/tree/main/community-containers#community-containers # APACHE_PORT: 11000 # Is needed when running behind a web server or reverse proxy (like Apache, Nginx, Caddy, Cloudflare Tunnel and else). See https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md # APACHE_IP_BINDING: 127.0.0.1 # Should be set when running behind a web server or reverse proxy (like Apache, Nginx, Caddy, Cloudflare Tunnel and else) that is running on the same host. See https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md + # APACHE_ADDITIONAL_NETWORK: frontend_net # (Optional) Connect the apache container to an additional docker network. Needed when behind a web server or reverse proxy (like Apache, Nginx, Caddy, Cloudflare Tunnel and else) running in a different docker network on same server. See https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md # BORG_RETENTION_POLICY: --keep-within=7d --keep-weekly=4 --keep-monthly=6 # Allows to adjust borgs retention policy. See https://github.com/nextcloud/all-in-one#how-to-adjust-borgs-retention-policy # COLLABORA_SECCOMP_DISABLED: false # Setting this to true allows to disable Collabora's Seccomp feature. See https://github.com/nextcloud/all-in-one#how-to-disable-collaboras-seccomp-feature # NEXTCLOUD_DATADIR: /mnt/ncdata # Allows to set the host directory for Nextcloud's datadir. ⚠️⚠️⚠️ Warning: do not set or adjust this value after the initial Nextcloud installation is done! See https://github.com/nextcloud/all-in-one#how-to-change-the-default-location-of-nextclouds-datadir diff --git a/php/src/Data/ConfigurationManager.php b/php/src/Data/ConfigurationManager.php index 527904fbfed..3b47b90cdd8 100644 --- a/php/src/Data/ConfigurationManager.php +++ b/php/src/Data/ConfigurationManager.php @@ -888,6 +888,13 @@ public function DeleteCollaboraDictionaries() : void { $this->WriteConfig($config); } + public function GetApacheAdditionalNetwork() : string { + $envVariableName = 'APACHE_ADDITIONAL_NETWORK'; + $configName = 'apache_additional_network'; + $defaultValue = ''; + return $this->GetEnvironmentalVariableOrConfig($envVariableName, $configName, $defaultValue); + } + public function GetApacheIPBinding() : string { $envVariableName = 'APACHE_IP_BINDING'; $configName = 'apache_ip_binding'; diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index 12a641e0869..bfd22c2a4ff 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -844,44 +844,49 @@ private function DisconnectContainerFromBridgeNetwork(string $id) : void } } - private function ConnectContainerIdToNetwork(string $id, string $internalPort, string $network = 'nextcloud-aio') : void + private function ConnectContainerIdToNetwork(string $id, string $internalPort, string $network = 'nextcloud-aio', bool $createNetwork = true, string $alias = '') : void { if ($internalPort === 'host') { return; } - $url = $this->BuildApiUrl('networks/create'); - try { - $this->guzzleClient->request( - 'POST', - $url, - [ - 'json' => [ - 'Name' => $network, - 'CheckDuplicate' => true, - 'Driver' => 'bridge', - 'Internal' => false, - ] - ] - ); - } catch (RequestException $e) { - // 409 is undocumented and gets thrown if the network already exists. - if ($e->getCode() !== 409) { - throw new \Exception("Could not create the nextcloud-aio network: " . $e->getMessage()); - } + if($createNetwork) { + $url = $this->BuildApiUrl('networks/create'); + try { + $this->guzzleClient->request( + 'POST', + $url, + [ + 'json' => [ + 'Name' => $network, + 'CheckDuplicate' => true, + 'Driver' => 'bridge', + 'Internal' => false, + ] + ] + ); + } catch (RequestException $e) { + // 409 is undocumented and gets thrown if the network already exists. + if ($e->getCode() !== 409) { + throw new \Exception("Could not create the nextcloud-aio network: " . $e->getMessage()); + } + } } $url = $this->BuildApiUrl( sprintf('networks/%s/connect', $network) ); + $json_payload = [ 'Container' => $id ]; + if ($alias !== '' ) { + $json_payload['EndpointConfig'] = [ 'Aliases' => [ $alias ] ]; + } + try { $this->guzzleClient->request( 'POST', $url, [ - 'json' => [ - 'container' => $id, - ] + 'json' => $json_payload ] ); } catch (RequestException $e) { @@ -897,11 +902,28 @@ public function ConnectMasterContainerToNetwork() : void $this->ConnectContainerIdToNetwork('nextcloud-aio-mastercontainer', ''); // Don't disconnect here since it slows down the initial login by a lot. Is getting done during cron.sh instead. // $this->DisconnectContainerFromBridgeNetwork('nextcloud-aio-mastercontainer'); + + $apacheAdditionalNetwork = $this->configurationManager->GetApacheAdditionalNetwork(); + if ($apacheAdditionalNetwork !== '') { + $this->ConnectContainerIdToNetwork('nextcloud-aio-mastercontainer', '', $apacheAdditionalNetwork, false); + } } public function ConnectContainerToNetwork(Container $container) : void { - $this->ConnectContainerIdToNetwork($container->GetIdentifier(), $container->GetInternalPort()); + // Add a secondary alias for domaincheck container, to keep it as similar to actual apache controller as possible. + // If a reverse-proxy is relying on container name as hostname this allows it to operate as usual and still validate the domain + // The domaincheck container and apache container are never supposed to be active at the same time because they use the same APACHE_PORT anyway, so this doesn't add any new constraints. + $alias = ($container->GetIdentifier() === 'nextcloud-aio-domaincheck') ? 'nextcloud-aio-apache' : ''; + + $this->ConnectContainerIdToNetwork($container->GetIdentifier(), $container->GetInternalPort(), alias: $alias); + + if ($container->GetIdentifier() === 'nextcloud-aio-apache' || $container->GetIdentifier() === 'nextcloud-aio-domaincheck') { + $apacheAdditionalNetwork = $this->configurationManager->GetApacheAdditionalNetwork(); + if ($apacheAdditionalNetwork !== '') { + $this->ConnectContainerIdToNetwork($container->GetIdentifier(), $container->GetInternalPort(), $apacheAdditionalNetwork, false, alias: $alias); + } + } } public function StopContainer(Container $container) : void { diff --git a/reverse-proxy.md b/reverse-proxy.md index 1345b5839ea..b58b5a20917 100644 --- a/reverse-proxy.md +++ b/reverse-proxy.md @@ -44,9 +44,10 @@ All examples below will use port `11000` as `APACHE_PORT`. This port will be exp On the same server in a Docker container - For this setup, you can use as target `host.docker.internal:$APACHE_PORT` instead of `localhost:$APACHE_PORT`. **⚠️ Important:** In order to make this work on Docker for Linux, you need to add `--add-host=host.docker.internal:host-gateway` to the docker run command of your reverse proxy container or `extra_hosts: ["host.docker.internal:host-gateway"]` in docker compose (it works on Docker Desktop by default). - - Another option (actually the recommended way) in this case is to use `--network host` option (or `network_mode: host` for docker-compose) as setting for the reverse proxy container to connect it to the host network. If you are using a firewall on the server, you need to open ports 80 and 443 for the reverse proxy manually. By doing so, the default sample configurations that point at `localhost:$APACHE_PORT` should work without having to modify them. + The reverse-proxy container needs to be connected to the nextcloud containers. This can be achieved one of these 3 ways: + 1. Utilize host networking instead of docker bridge networking: Specify `--network host` option (or `network_mode: host` for docker-compose) as setting for the reverse proxy container to connect it to the host network. If you are using a firewall on the server, you need to open ports 80 and 443 for the reverse proxy manually. With this setup, the default sample configurations with reverse-proxy pointing to `localhost:$APACHE_PORT` should work directly. + 1. Connect nextcloud's external-facing containers to the reverse-proxy's docker network by specifying env variable APACHE_ADDITIONAL_NETWORK. With this setup, the reverse proxy can utilize Docker bridge network's DNS name resolution to access nextcloud at `http://nextcloud-aio-apache:$APACHE_PORT`. Note, the specified network must already exist before Nextcloud AIO is started. + 1. Connect the reverse-proxy container to the `nextcloud-aio` network by specifying it as a secondary (external) network for the reverse proxy container. With this setup also, the reverse proxy can utilize Docker bridge network's DNS name resolution to access nextcloud at `http://nextcloud-aio-apache:$APACHE_PORT` . From 75c2407afa4be55087093aa6076ee01bd039743c Mon Sep 17 00:00:00 2001 From: Apoorv Parle <19315187+apparle@users.noreply.github.com> Date: Mon, 4 Nov 2024 02:44:10 -0800 Subject: [PATCH 2/6] Apply suggestions from code review Co-authored-by: Simon L. Signed-off-by: Apoorv Parle <19315187+apparle@users.noreply.github.com> --- php/src/Docker/DockerActionManager.php | 10 +++++----- reverse-proxy.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index bfd22c2a4ff..8cce610389f 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -850,7 +850,7 @@ private function ConnectContainerIdToNetwork(string $id, string $internalPort, s return; } - if($createNetwork) { + if ($createNetwork) { $url = $this->BuildApiUrl('networks/create'); try { $this->guzzleClient->request( @@ -876,9 +876,9 @@ private function ConnectContainerIdToNetwork(string $id, string $internalPort, s $url = $this->BuildApiUrl( sprintf('networks/%s/connect', $network) ); - $json_payload = [ 'Container' => $id ]; + $jsonPayload = [ 'Container' => $id ]; if ($alias !== '' ) { - $json_payload['EndpointConfig'] = [ 'Aliases' => [ $alias ] ]; + $jsonPayload['EndpointConfig'] = ['Aliases' => [ $alias ]]; } try { @@ -886,7 +886,7 @@ private function ConnectContainerIdToNetwork(string $id, string $internalPort, s 'POST', $url, [ - 'json' => $json_payload + 'json' => $jsonPayload ] ); } catch (RequestException $e) { @@ -921,7 +921,7 @@ public function ConnectContainerToNetwork(Container $container) : void if ($container->GetIdentifier() === 'nextcloud-aio-apache' || $container->GetIdentifier() === 'nextcloud-aio-domaincheck') { $apacheAdditionalNetwork = $this->configurationManager->GetApacheAdditionalNetwork(); if ($apacheAdditionalNetwork !== '') { - $this->ConnectContainerIdToNetwork($container->GetIdentifier(), $container->GetInternalPort(), $apacheAdditionalNetwork, false, alias: $alias); + $this->ConnectContainerIdToNetwork($container->GetIdentifier(), $container->GetInternalPort(), $apacheAdditionalNetwork, false, $alias); } } } diff --git a/reverse-proxy.md b/reverse-proxy.md index b58b5a20917..a7eea8c241c 100644 --- a/reverse-proxy.md +++ b/reverse-proxy.md @@ -46,7 +46,7 @@ All examples below will use port `11000` as `APACHE_PORT`. This port will be exp The reverse-proxy container needs to be connected to the nextcloud containers. This can be achieved one of these 3 ways: 1. Utilize host networking instead of docker bridge networking: Specify `--network host` option (or `network_mode: host` for docker-compose) as setting for the reverse proxy container to connect it to the host network. If you are using a firewall on the server, you need to open ports 80 and 443 for the reverse proxy manually. With this setup, the default sample configurations with reverse-proxy pointing to `localhost:$APACHE_PORT` should work directly. - 1. Connect nextcloud's external-facing containers to the reverse-proxy's docker network by specifying env variable APACHE_ADDITIONAL_NETWORK. With this setup, the reverse proxy can utilize Docker bridge network's DNS name resolution to access nextcloud at `http://nextcloud-aio-apache:$APACHE_PORT`. Note, the specified network must already exist before Nextcloud AIO is started. + 1. Connect nextcloud's external-facing containers to the reverse-proxy's docker network by specifying env variable APACHE_ADDITIONAL_NETWORK. With this setup, the reverse proxy can utilize Docker bridge network's DNS name resolution to access nextcloud at `http://nextcloud-aio-apache:$APACHE_PORT`. ⚠️⚠️⚠️ Note, the specified network must already exist before Nextcloud AIO is started. Otherwise it will fail to start the container because the network is not existing. 1. Connect the reverse-proxy container to the `nextcloud-aio` network by specifying it as a secondary (external) network for the reverse proxy container. With this setup also, the reverse proxy can utilize Docker bridge network's DNS name resolution to access nextcloud at `http://nextcloud-aio-apache:$APACHE_PORT` . From f5990cc0ef3f04a1efa5f2b7d02e3a98fd39d9b7 Mon Sep 17 00:00:00 2001 From: Apoorv Parle <19315187+apparle@users.noreply.github.com> Date: Mon, 4 Nov 2024 04:10:47 -0800 Subject: [PATCH 3/6] Add APACHE_ADDITIONAL_NETWORK variable to env variables QA test Signed-off-by: Apoorv Parle <19315187+apparle@users.noreply.github.com> --- tests/QA/060-environmental-variables.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/QA/060-environmental-variables.md b/tests/QA/060-environmental-variables.md index 488ae8e0d38..91b480f225e 100644 --- a/tests/QA/060-environmental-variables.md +++ b/tests/QA/060-environmental-variables.md @@ -2,6 +2,7 @@ - [ ] When starting the mastercontainer with `--env APACHE_PORT=11000` on a clean instance, the domaincheck container should be started with that same port published. That makes sure that also the Apache container will use that port later on. Using a value here that is not a port will not allow the mastercontainer to start correctly. - [ ] When starting the mastercontainer with `--env APACHE_IP_BINDING=127.0.0.1` on a clean instance, the domaincheck container's apache port should only listen on localhost on the host. Using a value here that is not a number or dot will not allow the mastercontainer to start correctly. +- [ ] When starting the mastercontainer with `--env APACHE_ADDITIONAL_NETWORK=frontend_net` on a clean instance, the domaincheck and subsequently the apache containers should be connected to the specified `frontend_net` docker network, in addition to the default `nextcloud-aio` network. Specifying the network that doesn't already exist will not allow the mastercontainer to start correctly. - [ ] When starting the mastercontainer with `--env TALK_PORT=3479` on a clean instance, the talk container should use this port later on. Using a value here that is not a port will not allow the mastercontainer to start correctly. Also it should stop if apache_port and talk_port are set to the same value. - [ ] Make also sure that reverse proxies work by following https://github.com/nextcloud/all-in-one/blob/main/reverse-proxy.md#reverse-proxy-documentation and following [001-initial-setup.md](./001-initial-setup.md) and [002-new-instance.md](./002-new-instance.md) - [ ] When starting the mastercontainer with `--env SKIP_DOMAIN_VALIDATION=true` on a clean instance, it should skip the domain verification. So it should accept any domain that you type in then. From 974ad1ef9b0875b0c62a8f266867724818aa038f Mon Sep 17 00:00:00 2001 From: Apoorv Parle <19315187+apparle@users.noreply.github.com> Date: Mon, 4 Nov 2024 05:27:30 -0800 Subject: [PATCH 4/6] Improve error checking and connect mastercontainer right from the start to make inital process seamless. Signed-off-by: Apoorv Parle <19315187+apparle@users.noreply.github.com> --- Containers/mastercontainer/start.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Containers/mastercontainer/start.sh b/Containers/mastercontainer/start.sh index 7dc94809abd..cbf79d753d5 100644 --- a/Containers/mastercontainer/start.sh +++ b/Containers/mastercontainer/start.sh @@ -200,6 +200,11 @@ It needs to be a string with letters, numbers, hyphens and underscores. It is set to '$APACHE_ADDITIONAL_NETWORK'." exit 1 fi + tmp_out=$(sudo -u www-data docker network connect $APACHE_ADDITIONAL_NETWORK nextcloud-aio-mastercontainer 2>&1) + if [ -n "$tmp_out" ] && [[ ! "$tmp_out" =~ "nextcloud-aio-mastercontainer already exists in network $APACHE_ADDITIONAL_NETWORK" ]]; then + print_red "Unable to connect to $APACHE_ADDITIONAL_NETWORK. Cannot continue. Error: \n $tmp_out" + exit 1 + fi fi if [ -n "$TALK_PORT" ]; then if ! check_if_number "$TALK_PORT"; then From e30b4562b26b88f7acfecdc79d34a44a16efbca7 Mon Sep 17 00:00:00 2001 From: Apoorv Parle <19315187+apparle@users.noreply.github.com> Date: Thu, 7 Nov 2024 00:26:35 -0800 Subject: [PATCH 5/6] Remove mastercontainer connection to APACHE_ADDITIONAL_CONTAINER. Follow that up in a separate PR. Signed-off-by: Apoorv Parle <19315187+apparle@users.noreply.github.com> --- Containers/mastercontainer/start.sh | 5 ----- php/src/Docker/DockerActionManager.php | 5 ----- 2 files changed, 10 deletions(-) diff --git a/Containers/mastercontainer/start.sh b/Containers/mastercontainer/start.sh index cbf79d753d5..7dc94809abd 100644 --- a/Containers/mastercontainer/start.sh +++ b/Containers/mastercontainer/start.sh @@ -200,11 +200,6 @@ It needs to be a string with letters, numbers, hyphens and underscores. It is set to '$APACHE_ADDITIONAL_NETWORK'." exit 1 fi - tmp_out=$(sudo -u www-data docker network connect $APACHE_ADDITIONAL_NETWORK nextcloud-aio-mastercontainer 2>&1) - if [ -n "$tmp_out" ] && [[ ! "$tmp_out" =~ "nextcloud-aio-mastercontainer already exists in network $APACHE_ADDITIONAL_NETWORK" ]]; then - print_red "Unable to connect to $APACHE_ADDITIONAL_NETWORK. Cannot continue. Error: \n $tmp_out" - exit 1 - fi fi if [ -n "$TALK_PORT" ]; then if ! check_if_number "$TALK_PORT"; then diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index 8cce610389f..0c4b00ecf63 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -902,11 +902,6 @@ public function ConnectMasterContainerToNetwork() : void $this->ConnectContainerIdToNetwork('nextcloud-aio-mastercontainer', ''); // Don't disconnect here since it slows down the initial login by a lot. Is getting done during cron.sh instead. // $this->DisconnectContainerFromBridgeNetwork('nextcloud-aio-mastercontainer'); - - $apacheAdditionalNetwork = $this->configurationManager->GetApacheAdditionalNetwork(); - if ($apacheAdditionalNetwork !== '') { - $this->ConnectContainerIdToNetwork('nextcloud-aio-mastercontainer', '', $apacheAdditionalNetwork, false); - } } public function ConnectContainerToNetwork(Container $container) : void From 4a8e0c8415e4c356333b0f19177a878f9414a052 Mon Sep 17 00:00:00 2001 From: "Simon L." Date: Thu, 7 Nov 2024 09:34:37 +0100 Subject: [PATCH 6/6] fix indentation Signed-off-by: Simon L. --- php/src/Docker/DockerActionManager.php | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/php/src/Docker/DockerActionManager.php b/php/src/Docker/DockerActionManager.php index 0c4b00ecf63..523aa2a1074 100644 --- a/php/src/Docker/DockerActionManager.php +++ b/php/src/Docker/DockerActionManager.php @@ -851,26 +851,26 @@ private function ConnectContainerIdToNetwork(string $id, string $internalPort, s } if ($createNetwork) { - $url = $this->BuildApiUrl('networks/create'); - try { - $this->guzzleClient->request( - 'POST', - $url, - [ - 'json' => [ - 'Name' => $network, - 'CheckDuplicate' => true, - 'Driver' => 'bridge', - 'Internal' => false, - ] + $url = $this->BuildApiUrl('networks/create'); + try { + $this->guzzleClient->request( + 'POST', + $url, + [ + 'json' => [ + 'Name' => $network, + 'CheckDuplicate' => true, + 'Driver' => 'bridge', + 'Internal' => false, ] - ); - } catch (RequestException $e) { - // 409 is undocumented and gets thrown if the network already exists. - if ($e->getCode() !== 409) { - throw new \Exception("Could not create the nextcloud-aio network: " . $e->getMessage()); - } + ] + ); + } catch (RequestException $e) { + // 409 is undocumented and gets thrown if the network already exists. + if ($e->getCode() !== 409) { + throw new \Exception("Could not create the nextcloud-aio network: " . $e->getMessage()); } + } } $url = $this->BuildApiUrl(