Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ReadMe.md
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ sftp:
password: "" # SFTP_PASSWORD
port: 22 # SFTP_PORT
key: "" # SFTP_KEY
known_hosts: "" # SFTP_KNOWN_HOSTS, path to OpenSSH known_hosts file; when set, the server host key is verified against it, when empty host key is not checked
path: "" # SFTP_PATH, `system.macros` values can be applied as {macro_name}
object_disk_path: "" # SFTP_OBJECT_DISK_PATH, path for backup of part from clickhouse object disks, if object disks present in clickhouse, then shall not be zero and shall not be prefixed by `path`
concurrency: 1 # SFTP_CONCURRENCY, default: (download_concurrency * 3)
Expand Down
12 changes: 7 additions & 5 deletions pkg/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -375,11 +375,13 @@ type FTPConfig struct {

// SFTPConfig - sftp settings section
type SFTPConfig struct {
Address string `yaml:"address" envconfig:"SFTP_ADDRESS"`
Port uint `yaml:"port" envconfig:"SFTP_PORT"`
Username string `yaml:"username" envconfig:"SFTP_USERNAME"`
Password string `yaml:"password" envconfig:"SFTP_PASSWORD"`
Key string `yaml:"key" envconfig:"SFTP_KEY"`
Address string `yaml:"address" envconfig:"SFTP_ADDRESS"`
Port uint `yaml:"port" envconfig:"SFTP_PORT"`
Username string `yaml:"username" envconfig:"SFTP_USERNAME"`
Password string `yaml:"password" envconfig:"SFTP_PASSWORD"`
Key string `yaml:"key" envconfig:"SFTP_KEY"`
// KnownHosts - path to an OpenSSH known_hosts file; when set, the server host key is verified against it (otherwise host keys are not checked)
KnownHosts string `yaml:"known_hosts" envconfig:"SFTP_KNOWN_HOSTS"`
Path string `yaml:"path" envconfig:"SFTP_PATH"`
ObjectDiskPath string `yaml:"object_disk_path" envconfig:"SFTP_OBJECT_DISK_PATH"`
CompressionFormat string `yaml:"compression_format" envconfig:"SFTP_COMPRESSION_FORMAT"`
Expand Down
50 changes: 50 additions & 0 deletions pkg/storage/sftp.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
libSFTP "github.com/pkg/sftp"
"github.com/rs/zerolog/log"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/knownhosts"
)

// SFTP Implement RemoteStorage
Expand Down Expand Up @@ -65,6 +66,18 @@ func (sftp *SFTP) Connect(_ context.Context) error {
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
if sftp.Config.KnownHosts != "" {
hostKeyCallback, err := knownhosts.New(sftp.Config.KnownHosts)
if err != nil {
return errors.Wrap(err, "SFTP Connect knownhosts.New")
}
hostKeyAlgorithms, err := knownHostsAlgorithms(sftp.Config.KnownHosts)
if err != nil {
return errors.Wrap(err, "SFTP Connect read known_hosts")
}
sftpConfig.HostKeyCallback = hostKeyCallback
sftpConfig.HostKeyAlgorithms = hostKeyAlgorithms
}
addr := fmt.Sprintf("%s:%d", sftp.Config.Address, sftp.Config.Port)
sftp.Debug("[SFTP_DEBUG] try connect to tcp://%s", addr)
sshConnection, err := ssh.Dial("tcp", addr, sftpConfig)
Expand Down Expand Up @@ -606,3 +619,40 @@ func (file *sftpFile) LastModified() time.Time {
func (file *sftpFile) Name() string {
return file.name
}

// knownHostsAlgorithms returns the host key algorithms for the key types in a
// known_hosts file, so the server offers a key the file can verify instead of
// failing with "key mismatch" on a different key type.
func knownHostsAlgorithms(path string) ([]string, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
seen := map[string]bool{}
algorithms := make([]string, 0)
add := func(algorithm string) {
if !seen[algorithm] {
seen[algorithm] = true
algorithms = append(algorithms, algorithm)
}
}
for len(data) > 0 {
var key ssh.PublicKey
_, _, key, _, data, err = ssh.ParseKnownHosts(data)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
if key.Type() == ssh.KeyAlgoRSA {
add(ssh.KeyAlgoRSASHA512)
add(ssh.KeyAlgoRSASHA256)
}
add(key.Type())
}
if len(algorithms) == 0 {
return nil, errors.Errorf("no host keys in %s", path)
}
return algorithms, nil
}
39 changes: 39 additions & 0 deletions pkg/storage/sftp_knownhosts_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package storage

import (
"os"
"path/filepath"
"testing"

"golang.org/x/crypto/ssh"
)

func TestKnownHostsAlgorithms(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "known_hosts")
content := "[box.example]:23 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl\n" +
"[box.example]:23 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQC6Mn1Pm6Vb2pWj1qhVGTPmkfB+VlRVbbyIA/sm0mtZOKvp5u0dKuOjc1gT0q6+JnkZ+PAgm3RP0pNC1mjIgkRgdWX8kKVuj3fy6/4+r+/KmS0j5yv0aQBUsBNG+SyexQBbyVN/zMjRfgO3oc6BuDZn4PZjnEVQaJmVdM3QDGXc0w==\n"
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
got, err := knownHostsAlgorithms(path)
if err != nil {
t.Fatal(err)
}
want := []string{ssh.KeyAlgoED25519, ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSA}
if len(got) != len(want) {
t.Fatalf("got %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("got %v, want %v", got, want)
}
}
empty := filepath.Join(dir, "empty")
if err := os.WriteFile(empty, []byte("# nothing\n"), 0o600); err != nil {
t.Fatal(err)
}
if _, err := knownHostsAlgorithms(empty); err == nil {
t.Fatal("expected error for a file without host keys")
}
}
Loading