Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,10 @@ Seamlessly terminate voice-over and lip-sync upon user interaction (e.g., skippi
![alt-text-1](./README/audio_dir.png "title-1")
- For your project's voice-over audio files, arrange them in the `audio/voice` directory.
- 🚨 Important: Ensure that the subdirectories under `audio/voice` have the same name with the character.
- 🚨 Important: Ensure that these audio files are in the `.wav` or `.ogg` format.
- 🚨 Important: Ensure that these audio files have `.wav` or `.ogg` suffixes.

### 5. Generate Lip Sync Data

- 🚨 Important: Before the execution, edit the `line 9` of `generate_lipsync_data.py`, according to your Rhubarb module name.
- Execute the `generate_lipsync_data.py` script located in the `lip-sync-plugin` directory. This script generates lip-sync data files for your audio files in the `lip-sync-data` subdirectory.
Run the following code in the `lip-sync-plugin` directory:
```bash
Expand Down
53 changes: 27 additions & 26 deletions game/lip-sync-plugin/generate_lipsync_data.py
Original file line number Diff line number Diff line change
@@ -1,31 +1,32 @@
import os
import subprocess
import re
from pathlib import Path

script_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(script_dir) # Get the parent directory of the script
script_dir = Path.cwd()
# Automatically find Rhubarb
rhubarb_path = [p for p in script_dir.glob("**/rhubarb") if p.is_file()][0]

def generate_lipsync_data(character_dir, audio_file):
# Generate lipsync data using Rhubarb (Note : Choose the appropriate Rhubarb version for your OS)
rhubarb_path = os.path.join(script_dir, "Rhubarb-Lip-Sync-1.13.0-macOS", "rhubarb") # change this to the appropriate path for your OS
audio_path = os.path.join(parent_dir, "audio", "voice", character_dir, audio_file)
output_path = os.path.join(script_dir, "lip-sync-data", character_dir, os.path.splitext(audio_file)[0] + ".txt")
# Create the corresponding directory structure in the output path
output_dir = os.path.dirname(output_path)
os.makedirs(output_dir, exist_ok=True)
# Run Rhubarb to generate lipsync data
subprocess.run([rhubarb_path, "-f", "tsv", "-o", output_path, audio_path], check=True)
print(f"Lipsync data generated for {audio_file}")

def trim_bracketed_strings(s):
# This regex will find all occurrences of {some string} in s
return re.sub(r'\{.*?\}', '', s).rstrip()
# Select English or Non-English
while True:
english = input("Are your voice recordings in English? (Y/N)")
if english in "yY":
command = [rhubarb_path, "-f", "tsv", "-o"]
break
elif english in "nN":
command = [rhubarb_path, "-r", "phonetic", "-f", "tsv", "-o"]
break
else:
continue

# Iterate through characters and voice files
characters = [d for d in os.listdir(os.path.join(script_dir, os.pardir, "audio", "voice")) if os.path.isdir(os.path.join(script_dir, os.pardir, "audio", "voice", d))]
for character in characters:
character = trim_bracketed_strings(character)
voice_dir = os.path.join(os.path.join(script_dir, os.pardir, "audio", "voice"), character)
voice_files = [f for f in os.listdir(voice_dir) if f.endswith(".wav") or f.endswith(".ogg")]
for voice_file in voice_files:
generate_lipsync_data(character, voice_file)
for voice_dir in (script_dir.parent / "audio" / "voice").iterdir():
if voice_dir.is_dir():
character = voice_dir.name
output_dir = script_dir / "lip-sync-data" / character
# Create the corresponding directory structure in the output path
output_dir.mkdir(exist_ok=True)
for audio_path in voice_dir.iterdir():
if audio_path.suffix in [".ogg", ".wav"] and audio_path.is_file():
output_path = (output_dir / audio_path.name).with_suffix(".txt")
process = command + [output_path, audio_path]
# Run Rhubarb to generate lipsync data
subprocess.run(process, check=True)