Skip to content

Commit c751964

Browse files
mitchsayremitchsayre
committed
Implement SpeechClient.generate() for ios
1 parent 56b4f43 commit c751964

12 files changed

Lines changed: 1900 additions & 439 deletions

File tree

‎android/src/main/java/com/wfloat/WfloatModule.kt‎

Lines changed: 15 additions & 241 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,9 @@
11
package com.wfloat
22

3-
import com.facebook.react.bridge.ReactApplicationContext
4-
import com.facebook.react.module.annotations.ReactModule
53
import com.facebook.react.bridge.Promise
4+
import com.facebook.react.bridge.ReactApplicationContext
65
import com.facebook.react.bridge.ReadableMap
7-
import android.content.res.AssetManager
8-
import com.k2fsa.sherpa.onnx.OfflineTts
9-
import com.k2fsa.sherpa.onnx.OfflineTtsConfig
10-
import com.k2fsa.sherpa.onnx.OfflineTtsModelConfig
11-
import com.k2fsa.sherpa.onnx.getOfflineTtsConfig
12-
import com.k2fsa.sherpa.onnx.OfflineTtsVitsModelConfig
13-
import android.util.Log
14-
import java.io.File
15-
import java.io.IOException
16-
import android.content.Context
17-
import java.io.FileOutputStream
18-
import java.io.InputStream
19-
import java.io.OutputStream
20-
import android.media.MediaPlayer
21-
import android.media.AudioAttributes
22-
import android.media.AudioFormat
23-
import android.media.AudioTrack
24-
import java.util.concurrent.LinkedBlockingQueue
6+
import com.facebook.react.module.annotations.ReactModule
257

268
@ReactModule(name = WfloatModule.NAME)
279
class WfloatModule(reactContext: ReactApplicationContext) :
@@ -38,233 +20,25 @@ class WfloatModule(reactContext: ReactApplicationContext) :
3820
)
3921
}
4022

41-
private fun copyDir(assetManager: AssetManager, assetDir: String, targetDir: File) {
42-
val assets = assetManager.list(assetDir) ?: return
43-
targetDir.mkdirs()
44-
for (asset in assets) {
45-
val path = "$assetDir/$asset"
46-
val file = File(targetDir, asset)
47-
if (assetManager.list(path)?.isNotEmpty() == true) {
48-
copyDir(assetManager, path, file)
49-
} else {
50-
assetManager.open(path).use { input: InputStream ->
51-
FileOutputStream(file).use { output: OutputStream ->
52-
input.copyTo(output)
53-
}
54-
}
55-
}
56-
}
57-
}
58-
59-
private fun copyAssetsToFilesDir(context: Context): Map<String, String> {
60-
val assetManager = context.assets
61-
val filesDir = context.filesDir
62-
63-
val espeakTarget = File(filesDir, "espeak-ng-data")
64-
copyDir(assetManager, "espeak-ng-data", espeakTarget)
65-
66-
val tokensTarget = File(filesDir, "tokens.txt")
67-
assetManager.open("tokens.txt").use { input: InputStream ->
68-
FileOutputStream(tokensTarget).use { output: OutputStream ->
69-
input.copyTo(output)
70-
}
71-
}
72-
73-
return mapOf(
74-
"espeakDirPath" to espeakTarget.absolutePath,
75-
"tokensFilePath" to tokensTarget.absolutePath
23+
override fun generate(options: ReadableMap, promise: Promise) {
24+
promise.reject(
25+
"UNIMPLEMENTED",
26+
"generate is only implemented on iOS right now."
7627
)
7728
}
7829

79-
override fun speech(modelPath: String, inputText: String): String {
80-
81-
val paths = copyAssetsToFilesDir(reactApplicationContext)
82-
val espeakDirPath = paths["espeakDirPath"]!!
83-
val tokensFilePath = paths["tokensFilePath"]!!
84-
85-
val fullModelPath = File(reactApplicationContext.filesDir, modelPath).absolutePath
86-
val fullModelFile = File(fullModelPath)
87-
val modelDirPath = fullModelFile.parent
88-
val modelName = fullModelFile.name
89-
90-
// Minimal dummy config — values can be empty since we just want to load JNI
91-
// val config = OfflineTtsVitsModelConfig(
92-
// model = fullModelPath,
93-
// tokens = tokensFilePath!!,
94-
// dataDir = espeakDirPath!!,
95-
// )
96-
97-
// val config = getOfflineTtsConfig(
98-
// modelDir = modelDirPath!!,
99-
// modelName = modelName,
100-
// acousticModelName = "",
101-
// vocoder = "",
102-
// voices = "",
103-
// lexicon = "",
104-
// dataDir = espeakDirPath!!,
105-
// dictDir = "",
106-
// ruleFsts = "",
107-
// ruleFars = "",
108-
// )
109-
110-
var assets = reactApplicationContext.assets
111-
112-
// val modelDir = File(modelDirPath)
113-
// val dataDir = File(espeakDirPath)
114-
// // val modelNameFile = File(modelName)
115-
//
116-
// val existenceReport = """
117-
// modelDir: ${modelDir.exists()}
118-
// fullModelFile: ${fullModelFile.exists()}
119-
// espeakDirPath: ${File(espeakDirPath).isDirectory()}
120-
//
121-
//""".trimIndent()
122-
123-
// return existenceReport
124-
125-
// val tts =
126-
// val tts = OfflineTts(config = config)
127-
128-
// val config = OfflineTtsConfig(
129-
// model = OfflineTtsVitsModelConfig(
130-
// model = fullModelPath, // should be the asset file name, e.g., "model.onnx"
131-
// tokens = tokensFilePath!!,
132-
// dataDir = espeakDirPath!!
133-
// )
134-
// )
135-
136-
val config = OfflineTtsConfig(
137-
model = OfflineTtsModelConfig(
138-
vits = OfflineTtsVitsModelConfig(
139-
model = fullModelPath,
140-
tokens = tokensFilePath,
141-
dataDir = espeakDirPath
142-
)
143-
)
144-
)
145-
val tts = OfflineTts(config = config)
146-
147-
val audio = tts.generate(
148-
text = inputText,
149-
sid = 0,
150-
speed = 1.0f
30+
override fun play(promise: Promise) {
31+
promise.reject(
32+
"UNIMPLEMENTED",
33+
"play is only implemented on iOS right now."
15134
)
152-
153-
val tempDirPath = reactApplicationContext.cacheDir.absolutePath
154-
val timestamp = (System.currentTimeMillis() / 1000).toString()
155-
val filePath = "$tempDirPath/audio_$timestamp.wav"
156-
157-
// val filename = "audio.wav"
158-
val result = audio.save(filename = filePath)
159-
tts.free()
160-
161-
return filePath
162-
// return "${espeakDirPath};${tokensFilePath};${modelName};${modelDirPath}" // This will crash if JNI isn't properly loaded
16335
}
16436

165-
override fun streamSpeech(modelPath: String, inputText: String, promise: Promise) {
166-
try {
167-
val paths = copyAssetsToFilesDir(reactApplicationContext)
168-
val espeakDirPath = paths["espeakDirPath"]!!
169-
val tokensFilePath = paths["tokensFilePath"]!!
170-
171-
val fullModelPath = File(reactApplicationContext.filesDir, modelPath).absolutePath
172-
val fullModelFile = File(fullModelPath)
173-
174-
val config = OfflineTtsConfig(
175-
model = OfflineTtsModelConfig(
176-
vits = OfflineTtsVitsModelConfig(
177-
model = fullModelPath,
178-
tokens = tokensFilePath,
179-
dataDir = espeakDirPath
180-
)
181-
)
182-
)
183-
184-
val tts = OfflineTts(config = config)
185-
186-
val sampleRate = tts.sampleRate()
187-
val bufferSize = AudioTrack.getMinBufferSize(
188-
sampleRate,
189-
AudioFormat.CHANNEL_OUT_MONO,
190-
AudioFormat.ENCODING_PCM_FLOAT
191-
)
192-
193-
val audioTrack = AudioTrack.Builder()
194-
.setAudioAttributes(
195-
AudioAttributes.Builder()
196-
.setUsage(AudioAttributes.USAGE_MEDIA)
197-
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
198-
.build()
199-
)
200-
.setAudioFormat(
201-
AudioFormat.Builder()
202-
.setEncoding(AudioFormat.ENCODING_PCM_FLOAT)
203-
.setSampleRate(sampleRate)
204-
.setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
205-
.build()
206-
)
207-
.setBufferSizeInBytes(bufferSize)
208-
.setTransferMode(AudioTrack.MODE_STREAM)
209-
.build()
210-
211-
val audioQueue = LinkedBlockingQueue<FloatArray>()
212-
213-
val writerThread = Thread {
214-
audioTrack.play()
215-
while (true) {
216-
val samples = audioQueue.take() // blocks until data is available
217-
if (samples.isEmpty()) break // poison pill to stop
218-
audioTrack.write(samples, 0, samples.size, AudioTrack.WRITE_BLOCKING)
219-
}
220-
}
221-
writerThread.start()
222-
223-
val audio = tts.generateWithCallback(
224-
text = inputText,
225-
sid = 0,
226-
speed = 1.0f,
227-
callback = { samples: FloatArray ->
228-
audioQueue.put(samples)
229-
1
230-
}
231-
)
232-
233-
// After generation is done, signal the writer thread to stop
234-
audioQueue.put(FloatArray(0))
235-
writerThread.join()
236-
237-
val tempDirPath = reactApplicationContext.cacheDir.absolutePath
238-
val timestamp = (System.currentTimeMillis() / 1000).toString()
239-
val filePath = "$tempDirPath/audio_$timestamp.wav"
240-
241-
audio.save(filename = filePath)
242-
243-
tts.free()
244-
audioTrack.stop()
245-
audioTrack.release()
246-
247-
promise.resolve(filePath)
248-
} catch (e: Exception) {
249-
Log.e(NAME, "streamSpeech failed", e)
250-
promise.reject("STREAM_ERROR", "Failed to stream speech", e)
251-
}
252-
}
253-
254-
override fun playWav(filePath: String): String {
255-
256-
// val assetFileDescriptor = reactApplicationContext.assets.openFd("speaker_0.wav")
257-
//
258-
val mediaPlayer = MediaPlayer()
259-
mediaPlayer.setDataSource(filePath)
260-
// mediaPlayer.setDataSource(
261-
// assetFileDescriptor.fileDescriptor,
262-
// assetFileDescriptor.startOffset,
263-
// assetFileDescriptor.length
264-
// )
265-
mediaPlayer.prepare()
266-
mediaPlayer.start()
267-
return "success"
37+
override fun pause(promise: Promise) {
38+
promise.reject(
39+
"UNIMPLEMENTED",
40+
"pause is only implemented on iOS right now."
41+
)
26842
}
26943

27044
companion object {

0 commit comments

Comments
 (0)