Skip to content

Commit 0d433b4

Browse files
authored
Merge pull request #1660 from maratsh/sftp-known-hosts
sftp: optional known_hosts host key verification
2 parents bdc35ec + ea4c97e commit 0d433b4

5 files changed

Lines changed: 98 additions & 6 deletions

File tree

‎ReadMe.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -519,6 +519,7 @@ sftp:
519519
password: "" # SFTP_PASSWORD
520520
port: 22 # SFTP_PORT
521521
key: "" # SFTP_KEY
522+
known_hosts: "" # SFTP_KNOWN_HOSTS, path to OpenSSH known_hosts file; when set, the server host key is verified against it, when empty host key is not checked
522523
path: "" # SFTP_PATH, `system.macros` values can be applied as {macro_name}
523524
object_disk_path: "" # SFTP_OBJECT_DISK_PATH, path for backup of part from clickhouse object disks, if object disks present in clickhouse, then shall not be zero and shall not be prefixed by `path`
524525
concurrency: 1 # SFTP_CONCURRENCY, default: (download_concurrency * 3)

‎pkg/config/config.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -375,11 +375,13 @@ type FTPConfig struct {
375375

376376
// SFTPConfig - sftp settings section
377377
type SFTPConfig struct {
378-
Address string `yaml:"address" envconfig:"SFTP_ADDRESS"`
379-
Port uint `yaml:"port" envconfig:"SFTP_PORT"`
380-
Username string `yaml:"username" envconfig:"SFTP_USERNAME"`
381-
Password string `yaml:"password" envconfig:"SFTP_PASSWORD"`
382-
Key string `yaml:"key" envconfig:"SFTP_KEY"`
378+
Address string `yaml:"address" envconfig:"SFTP_ADDRESS"`
379+
Port uint `yaml:"port" envconfig:"SFTP_PORT"`
380+
Username string `yaml:"username" envconfig:"SFTP_USERNAME"`
381+
Password string `yaml:"password" envconfig:"SFTP_PASSWORD"`
382+
Key string `yaml:"key" envconfig:"SFTP_KEY"`
383+
// KnownHosts - path to an OpenSSH known_hosts file; when set, the server host key is verified against it (otherwise host keys are not checked)
384+
KnownHosts string `yaml:"known_hosts" envconfig:"SFTP_KNOWN_HOSTS"`
383385
Path string `yaml:"path" envconfig:"SFTP_PATH"`
384386
ObjectDiskPath string `yaml:"object_disk_path" envconfig:"SFTP_OBJECT_DISK_PATH"`
385387
CompressionFormat string `yaml:"compression_format" envconfig:"SFTP_COMPRESSION_FORMAT"`

‎pkg/storage/sftp.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
libSFTP "github.com/pkg/sftp"
1818
"github.com/rs/zerolog/log"
1919
"golang.org/x/crypto/ssh"
20+
"golang.org/x/crypto/ssh/knownhosts"
2021
)
2122

2223
// SFTP Implement RemoteStorage
@@ -65,6 +66,18 @@ func (sftp *SFTP) Connect(_ context.Context) error {
6566
Auth: authMethods,
6667
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
6768
}
69+
if sftp.Config.KnownHosts != "" {
70+
hostKeyCallback, err := knownhosts.New(sftp.Config.KnownHosts)
71+
if err != nil {
72+
return errors.Wrap(err, "SFTP Connect knownhosts.New")
73+
}
74+
hostKeyAlgorithms, err := knownHostsAlgorithms(sftp.Config.KnownHosts)
75+
if err != nil {
76+
return errors.Wrap(err, "SFTP Connect read known_hosts")
77+
}
78+
sftpConfig.HostKeyCallback = hostKeyCallback
79+
sftpConfig.HostKeyAlgorithms = hostKeyAlgorithms
80+
}
6881
addr := fmt.Sprintf("%s:%d", sftp.Config.Address, sftp.Config.Port)
6982
sftp.Debug("[SFTP_DEBUG] try connect to tcp://%s", addr)
7083
sshConnection, err := ssh.Dial("tcp", addr, sftpConfig)
@@ -606,3 +619,40 @@ func (file *sftpFile) LastModified() time.Time {
606619
func (file *sftpFile) Name() string {
607620
return file.name
608621
}
622+
623+
// knownHostsAlgorithms returns the host key algorithms for the key types in a
624+
// known_hosts file, so the server offers a key the file can verify instead of
625+
// failing with "key mismatch" on a different key type.
626+
func knownHostsAlgorithms(path string) ([]string, error) {
627+
data, err := os.ReadFile(path)
628+
if err != nil {
629+
return nil, err
630+
}
631+
seen := map[string]bool{}
632+
algorithms := make([]string, 0)
633+
add := func(algorithm string) {
634+
if !seen[algorithm] {
635+
seen[algorithm] = true
636+
algorithms = append(algorithms, algorithm)
637+
}
638+
}
639+
for len(data) > 0 {
640+
var key ssh.PublicKey
641+
_, _, key, _, data, err = ssh.ParseKnownHosts(data)
642+
if err == io.EOF {
643+
break
644+
}
645+
if err != nil {
646+
return nil, err
647+
}
648+
if key.Type() == ssh.KeyAlgoRSA {
649+
add(ssh.KeyAlgoRSASHA512)
650+
add(ssh.KeyAlgoRSASHA256)
651+
}
652+
add(key.Type())
653+
}
654+
if len(algorithms) == 0 {
655+
return nil, errors.Errorf("no host keys in %s", path)
656+
}
657+
return algorithms, nil
658+
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package storage
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"golang.org/x/crypto/ssh"
9+
)
10+
11+
func TestKnownHostsAlgorithms(t *testing.T) {
12+
dir := t.TempDir()
13+
path := filepath.Join(dir, "known_hosts")
14+
content := "[box.example]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n" +
15+
"[box.example]:23 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Mn1Pm6Vb2pWj1qhVGTPmkfB+VlRVbbyIA/sm0mtZOKvp5u0dKuOjc1gT0q6+JnkZ+PAgm3RP0pNC1mjIgkRgdWX8kKVuj3fy6/4+r+/KmS0j5yv0aQBUsBNG+SyexQBbyVN/zMjRfgO3oc6BuDZn4PZjnEVQaJmVdM3QDGXc0w==\n"
16+
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
17+
t.Fatal(err)
18+
}
19+
got, err := knownHostsAlgorithms(path)
20+
if err != nil {
21+
t.Fatal(err)
22+
}
23+
want := []string{ssh.KeyAlgoED25519, ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA}
24+
if len(got) != len(want) {
25+
t.Fatalf("got %v, want %v", got, want)
26+
}
27+
for i := range want {
28+
if got[i] != want[i] {
29+
t.Fatalf("got %v, want %v", got, want)
30+
}
31+
}
32+
empty := filepath.Join(dir, "empty")
33+
if err := os.WriteFile(empty, []byte("# nothing\n"), 0o600); err != nil {
34+
t.Fatal(err)
35+
}
36+
if _, err := knownHostsAlgorithms(empty); err == nil {
37+
t.Fatal("expected error for a file without host keys")
38+
}
39+
}

0 commit comments

Comments
 (0)