From bded0b93499dca7f1922ff6c6222cf99a197fda7 Mon Sep 17 00:00:00 2001 From: Colton Loftus <70598503+C-Loftus@users.noreply.github.com> Date: Sat, 12 Oct 2024 18:09:04 -0400 Subject: [PATCH] clean ups --- .vscode/launch.json | 15 ++++++++--- Dockerfile | 5 ++-- cli/cli.go | 63 ++++++++++++++++++++------------------------ cli/e2e_test.go | 31 +++++++++++++++++++++- lib/models.go | 11 +++++--- lib/models_test.go | 51 +++++++++++++++++++++++++++++++++++ lib/test_chinese.txt | 1 + readme.md | 1 - 8 files changed, 133 insertions(+), 45 deletions(-) create mode 100644 lib/models_test.go create mode 100644 lib/test_chinese.txt diff --git a/.vscode/launch.json b/.vscode/launch.json index 5e79418..3d27264 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,13 +4,22 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { - "name": "Launch Package", + "name": "Convert Remote", "type": "go", "request": "launch", "mode": "auto", - "program": "${fileDirname}", + "program": "${workspaceFolder}", "args": ["https://example-files.online-convert.com/document/txt/example.txt"], - } + }, + { + "name": "Convert Chinese", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}", + "args": ["--model=zh_CN-huayan-medium.onnx", "lib/test_chinese.txt", "--speak-utf-8"] + }, ] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d846e94..3fc563d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -## This dockerfile is primarily for testing. It can be used like the following: +# This dockerfile can be used to build a binary for use with the QuickPiperAudiobook command. +# You can use it for testing, or other architectures that don't have a piper build. # docker build -t quickpiperaudiobook . # docker run quickpiperaudiobook /app/examples/lorem_ipsum.txt @@ -6,14 +7,12 @@ FROM --platform=linux/amd64 golang:1.22 as build WORKDIR /app -# Copy all the code from the current directory COPY . . # Install Go dependencies and build the binary RUN go mod tidy && \ go build -o QuickPiperAudiobook . -# Final stage FROM --platform=linux/amd64 ubuntu:latest # Install runtime dependencies diff --git a/cli/cli.go b/cli/cli.go index ae712a1..6af2acb 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -14,11 +14,11 @@ import ( ) type CLI struct { - Input string `arg:"" help:"Local path or URL to the input file"` - Output string `help:"Directory in which to save the converted ebook file"` - Model string `help:"Local path to the onnx model for piper to use"` - SpeakDiacritics bool `help:"Speak diacritics from the input file"` - ListModels bool `help:"List available models"` + Input string `arg:"" help:"Local path or URL to the input file"` + Output string `help:"The directory in which to save the output audiobook"` + Model string `help:"Local path to the onnx model for piper to use"` + SpeakUTF8 bool `help:"Speak UTF-8 characters; Necessary for many non-English languages."` + ListModels bool `help:"List piper models which are installed locally"` } // package level variables we want to expose for testing @@ -32,15 +32,15 @@ const defaultModel = "en_US-hfc_male-medium.onnx" func RunCLI() { if homedir_err != nil { - fmt.Printf("Error getting user home directory: %v\n", homedir_err) - return + fmt.Fprintf(os.Stderr, "Error getting user home directory: %v\n", homedir_err) + os.Exit(1) } var config CLI if err := lib.CreateConfigIfNotExists(configFile, configDir, defaultModel); err != nil { - fmt.Printf("Error: %v\n", err) - return + fmt.Fprintf(os.Stderr, "Error creating default config file: %v\n", err) + os.Exit(1) } parser, _ := kong.New(&config, kong.Configuration(kongyaml.Loader, configFile)) @@ -49,8 +49,8 @@ func RunCLI() { _, err := parser.Parse([]string{name}) if err != nil { - fmt.Println("Error parsing the value for", name, "in your config file at:", configFile) - return + fmt.Fprintf(os.Stderr, "Error parsing the value for %s in your config file at: %s\n", name, configFile) + os.Exit(1) } } @@ -60,9 +60,7 @@ func RunCLI() { if cli.ListModels { models, err := lib.FindModels(configDir) if err != nil { - fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) - return } if len(models) == 0 { @@ -74,7 +72,7 @@ func RunCLI() { } if cli.Output == "" && config.Output != "" { - fmt.Println("No output value specified, default from config file: " + config.Output) + fmt.Println("No output directory specified, default from config file: " + config.Output) cli.Output = config.Output // if output is not set and config is not set, default to current directory } else if cli.Output == "" && config.Output == "" { @@ -98,9 +96,7 @@ func RunCLI() { if _, err := os.Stat(cli.Output); os.IsNotExist(err) { err := os.MkdirAll(cli.Output, os.ModePerm) if err != nil { - fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) - return } } @@ -109,55 +105,54 @@ func RunCLI() { if err := lib.CheckEbookConvertInstalled(); err != nil { fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) - return } } if !lib.PiperIsInstalled(configDir) { if err := lib.InstallPiper(configDir); err != nil { ctx.FatalIfErrorf(err) - return } } else { slog.Debug("Piper install detected in " + configDir) } - modelPath, err := lib.ExpandModelPath(cli.Model, configDir) + modelPath, modelPathErr := lib.ExpandModelPath(cli.Model, configDir) - if err != nil { + // Some errors above are fine; (we can just download the corresponding model) + // but others that pertain to having the model but not the corresponding metadata are + // an error that should be fatal + if modelPathErr != nil && strings.Contains(modelPathErr.Error(), "but the corresponding") { + ctx.FatalIfErrorf(modelPathErr) + } + + if modelPathErr != nil { // if the path can't be expanded, it doesn't exist and we need to download it err := lib.DownloadModelIfNotExists(cli.Model, configDir) + + if err != nil && modelPathErr != nil { + fmt.Printf("Error: %v\n", modelPathErr) + } + if err != nil { - fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) - return } modelPath, err = lib.ExpandModelPath(cli.Model, configDir) - if err != nil { - fmt.Printf("Error could not find the model path after downloading it: %v\n", err) - ctx.FatalIfErrorf(err) - return + ctx.FatalIfErrorf(fmt.Errorf("error could not find the model path after downloading it: %v", err)) } } data, err := lib.GetConvertedRawText(cli.Input) if err != nil { - fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) } else if data == nil { - fmt.Println("After converting" + cli.Input + "to txt, no data was generated.") - return - } else { - fmt.Println("Text conversion completed successfully.") + ctx.FatalIfErrorf(fmt.Errorf("after converting %s to txt, no data was generated", cli.Input)) } - if !cli.SpeakDiacritics { + if !cli.SpeakUTF8 { if data, err = lib.RemoveDiacritics(data); err != nil { - fmt.Printf("Error: %v\n", err) ctx.FatalIfErrorf(err) - return } } diff --git a/cli/e2e_test.go b/cli/e2e_test.go index 390b8c1..f2e2c9e 100644 --- a/cli/e2e_test.go +++ b/cli/e2e_test.go @@ -2,6 +2,7 @@ package cli import ( "os" + "path/filepath" "testing" ) @@ -17,10 +18,38 @@ func TestCLIWithDiacritics(t *testing.T) { // reset all cli args, since the golang testing framework changes them os.RemoveAll(configDir) origArgs := os.Args - os.Args = append(origArgs[:1], "https://example-files.online-convert.com/document/txt/example.txt", "--speak-diacritics") + os.Args = append(origArgs[:1], "https://example-files.online-convert.com/document/txt/example.txt", "--speak-utf-8") RunCLI() // make sure that after running you can run the list models command and it will work os.Args = append(origArgs[:1], "https://example-files.online-convert.com/document/txt/example.txt", "--list-models") RunCLI() } + +// Test that the cli works with chinese language text +func TestChinese(t *testing.T) { + // reset all cli args, since the golang testing framework changes them + os.RemoveAll(configDir) + origArgs := os.Args + // get the file located at ../lib/test_chinese.txt + os.Args = append(origArgs[:1], "../lib/test_chinese.txt", "--model=zh_CN-huayan-medium.onnx", "--speak-utf-8") + RunCLI() + + // check if there is a file at ~/Audiobooks/test_chinese.wav and make sure it's not empty + homedir, err := os.UserHomeDir() + if err != nil { + t.Fatalf("error getting user home directory: %v", err) + } + testFile := filepath.Join(homedir, "Audiobooks", "test_chinese.wav") + defer os.Remove(testFile) + + if info, err := os.Stat(testFile); err != nil { + if os.IsNotExist(err) { + t.Fatalf("file not created: %v", err) + } + t.Fatalf("error getting file info: %v", err) + } else if info.Size() == 0 { + t.Fatalf("file is empty") + } + +} diff --git a/lib/models.go b/lib/models.go index bb442d1..2dca52b 100644 --- a/lib/models.go +++ b/lib/models.go @@ -15,6 +15,10 @@ var ModelToURL = map[string]string{ "en_US-hfc_female-medium.onnx": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/hfc_female/medium/en_US-hfc_female-medium.onnx", "en_US-lessac-medium.onnx": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_US/lessac/medium/en_US-lessac-medium.onnx", "en_GB-northern_english_male-medium.onnx": "https://huggingface.co/rhasspy/piper-voices/resolve/main/en/en_GB/northern_english_male/medium/en_GB-northern_english_male-medium.onnx", + // Below is an example of a non-English model + // I happily accept PRs for others here. It is just a bit tedious to enumerate them all + // since some do not follow the same pattern. + "zh_CN-huayan-medium.onnx": "https://huggingface.co/rhasspy/piper-voices/resolve/main/zh/zh_CN/huayan/medium/zh_CN-huayan-medium.onnx", } func ExpandModelPath(modelName string, defaultModelDir string) (string, error) { @@ -25,14 +29,15 @@ func ExpandModelPath(modelName string, defaultModelDir string) (string, error) { if _, err := os.Stat(modelName + ".json"); err == nil { return modelName, nil } - return "", fmt.Errorf("onnx for model: %s was found but the corresponding onnx.json was not", modelName) + return "", fmt.Errorf("onnx for model '%s' was found but the corresponding onnx.json was not", modelName) } + if _, err := os.Stat(filepath.Join(defaultModelDir, modelName)); err == nil { if _, err := os.Stat(filepath.Join(defaultModelDir, modelName) + ".json"); err == nil { return filepath.Join(defaultModelDir, modelName), nil } - return "", fmt.Errorf("onnx for model: %s was found in the model directory: %s but the corresponding onnx.json was not", modelName, defaultModelDir) + return "", fmt.Errorf("onnx for model '%s' was found in the model directory: '%s' but the corresponding onnx.json was not", modelName, defaultModelDir) } - return "", fmt.Errorf("model not found: %s", modelName) + return "", fmt.Errorf("model: %s", modelName) } diff --git a/lib/models_test.go b/lib/models_test.go new file mode 100644 index 0000000..d6e7bb1 --- /dev/null +++ b/lib/models_test.go @@ -0,0 +1,51 @@ +package lib + +import ( + "os" + "path/filepath" + "testing" +) + +func TestExpandModelPath(t *testing.T) { + // Create a temporary directory for testing + tempDir := t.TempDir() + modelName := "test_model" + modelPath := filepath.Join(tempDir, modelName) + modelJSONPath := modelPath + ".json" + + // Test case 1: Both ONNX and JSON files are present + os.WriteFile(modelPath, []byte("dummy ONNX model"), 0644) + os.WriteFile(modelJSONPath, []byte("dummy JSON"), 0644) + + result, err := ExpandModelPath(modelName, tempDir) + if err != nil || result != modelPath { + t.Errorf("Expected %s, got %s, error: %v", modelPath, result, err) + } + + // Test case 2: ONNX file is present, but JSON file is missing + os.Remove(modelJSONPath) // remove the JSON file + + result, err = ExpandModelPath(modelName, tempDir) + if err == nil || result != "" { + t.Errorf("Expected error for missing JSON file, got: %v, result: %s", err, result) + } + + // Test case 3: Model not found + result, err = ExpandModelPath("non_existent_model", tempDir) + if err == nil || result != "" { + t.Errorf("Expected error for non-existent model, got: %v, result: %s", err, result) + } + + // Test case 4: Model found in the default model directory + modelNameInDir := "another_model" + modelPathInDir := filepath.Join(tempDir, modelNameInDir) + modelJSONPathInDir := modelPathInDir + ".json" + + os.WriteFile(modelPathInDir, []byte("dummy ONNX model"), 0644) + os.WriteFile(modelJSONPathInDir, []byte("dummy JSON"), 0644) + + result, err = ExpandModelPath(modelNameInDir, tempDir) + if err != nil || result != modelPathInDir { + t.Errorf("Expected %s, got %s, error: %v", modelPathInDir, result, err) + } +} diff --git a/lib/test_chinese.txt b/lib/test_chinese.txt new file mode 100644 index 0000000..481015a --- /dev/null +++ b/lib/test_chinese.txt @@ -0,0 +1 @@ +彩虹，又稱天弓、天虹、絳等，簡稱虹，是氣象中的一種光學現象，當太陽 光照射到半空中的水滴，光線被折射及反射，在天空上形成拱形的七彩光譜，由外 圈至内圈呈紅、橙、黃、綠、蓝、靛蓝、堇紫七种颜色（霓虹則相反）。 \ No newline at end of file diff --git a/readme.md b/readme.md index fbafd5f..cee4653 100644 --- a/readme.md +++ b/readme.md @@ -19,7 +19,6 @@ Listen to sample output [ here ](./examples/) > You don't need to have piper installed. This program manages piper and the associated models - ## Usage * Pass in either a local file or a remote URL to generate an audiobook: