audiobook: support reading from audio files in TTS

android/debug/org.coolreader.crengine.L.java

@@ -0,0 +1,75 @@
+package org.coolreader.crengine;
+
+public class L {
+  public static class LoggerImpl implements Logger {
+    public void i(String msg) {
+      System.out.println(msg);
+    }
+    public void i(String msg, Exception e) {
+      System.out.println(msg);
+    }
+    public void w(String msg) {
+      System.out.println(msg);
+    }
+    public void w(String msg, Exception e) {
+      System.out.println(msg);
+    }
+    public void e(String msg) {
+      System.out.println(msg);
+    }
+    public void e(String msg, Exception e) {
+      System.out.println(msg);
+    }
+    public void d(String msg) {
+      System.out.println(msg);
+    }
+    public void d(String msg, Exception e) {
+      System.out.println(msg);
+    }
+    public void v(String msg) {
+      System.out.println(msg);
+    }
+    public void v(String msg, Exception e) {
+      System.out.println(msg);
+    }
+    public void setLevel(int level) {
+    }
+  }
+
+  public static void i(String msg) {
+    System.out.println(msg);
+  }
+  public static void i(String msg, Exception e) {
+    System.out.println(msg);
+  }
+  public static void w(String msg) {
+    System.out.println(msg);
+  }
+  public static void w(String msg, Exception e) {
+    System.out.println(msg);
+  }
+  public static void e(String msg) {
+    System.out.println(msg);
+  }
+  public static void e(String msg, Exception e) {
+    System.out.println(msg);
+  }
+  public static void d(String msg) {
+    System.out.println(msg);
+  }
+  public static void d(String msg, Exception e) {
+    System.out.println(msg);
+  }
+  public static void v(String msg) {
+    System.out.println(msg);
+  }
+  public static void v(String msg, Exception e) {
+    System.out.println(msg);
+  }
+  public static Logger create(String name) {
+    return new LoggerImpl();
+  }
+  public static Logger create(String name, int level) {
+    return new LoggerImpl();
+  }
+}

android/jni/docview.cpp

@@ -1808,6 +1808,64 @@ JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getCurrentPageBoo
 	return obj;
 }
 
+/*
+ * Class:     org_coolreader_crengine_DocView
+ * Method:    getAllSentencesInternal
+ * Signature: ()Ljava/util/ArrayList;
+ */
+JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getAllSentencesInternal
+    (JNIEnv * _env, jobject _this)
+{
+    CRJNIEnv env(_env);
+    DocViewNative * p = getNative(_env, _this);
+    if (!p) {
+        CRLog::error("Cannot get native view");
+        return NULL;
+    }
+
+    jclass arrListClass = _env->FindClass("java/util/ArrayList");
+    jmethodID arrListCtor = _env->GetMethodID(arrListClass, "<init>", "()V");
+    jmethodID arrListAdd = env->GetMethodID(arrListClass, "add", "(Ljava/lang/Object;)Z");
+
+    jclass sentenceInfoClass = _env->FindClass("org/coolreader/crengine/SentenceInfo");
+    jmethodID sentenceInfoCtor = _env->GetMethodID(sentenceInfoClass, "<init>", "()V");
+    jmethodID sentenceInfoSetText = _env->GetMethodID(sentenceInfoClass, "setText", "(Ljava/lang/String;)V");
+    jmethodID sentenceInfoSetStartPos = _env->GetMethodID(sentenceInfoClass, "setStartPos", "(Ljava/lang/String;)V");
+
+    jobject arrList = env->NewObject(arrListClass, arrListCtor);
+
+
+    p->_docview->savePosition();
+    p->_docview->clearSelection();
+    p->_docview->goToPage(0);
+    p->_docview->SetPos(0, false);
+    while(p->_docview->nextSentence()){
+        jobject sentenceInfo = _env->NewObject(sentenceInfoClass, sentenceInfoCtor);
+        jint startX = 0;
+        jint startY = 0;
+
+        ldomXRangeList & sel = p->_docview->getDocument()->getSelections();
+        ldomXRange currSel;
+        if ( sel.length()>0 ){
+            currSel = *sel[0];
+        }
+        lvPoint startPoint = currSel.getStart().toPoint();
+        lvPoint endPoint = currSel.getEnd().toPoint();
+
+        env->CallVoidMethod(sentenceInfo, sentenceInfoSetText, env->NewStringUTF(
+          UnicodeToUtf8(currSel.getRangeText()).c_str()
+        ));
+        env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartPos, env->NewStringUTF(
+          UnicodeToUtf8(currSel.getStart().toString()).c_str()
+        ));
+
+        env->CallBooleanMethod(arrList, arrListAdd, sentenceInfo);
+    }
+    p->_docview->restorePosition();
+
+    return arrList;
+}
+
 /*
  * Class:     org_coolreader_crengine_DocView
  * Method:    updateBookInfoInternal

android/res/values/strings.xml

@@ -553,6 +553,7 @@
     <string name="options_tts_voice_quality_very_high">Very high quality</string>
     <string name="options_tts_google_abbr_workaround">Use a workaround to disable processing of abbreviations at the end of a sentence</string>
     <string name="options_tts_google_abbr_workaround_comment">when using "Google Speech Services"</string>
+    <string name="options_tts_use_audiobook">Use audiobook instead of TTS, if *.wordtiming file exists</string>
     <string name="tts_init_failure">TTS engine \"%s\" init failure, disabling it…</string>
     <string name="dlg_button_open_book">Open book</string>
     <string name="dlg_button_open_folder">Open containing folder</string>

android/run-wordtiming-main.pl

@@ -0,0 +1,46 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+
+my @classes = qw(
+  org/coolreader/crengine/WordTimingAudiobookMatcher.java
+  org/coolreader/crengine/SentenceInfo.java
+  org/coolreader/crengine/L.java
+  org/coolreader/crengine/Logger.java
+  org/coolreader/crengine/SentenceInfoCache.java
+);
+
+sub main(@){
+  system "rm", "-rf", "run-main-tmp/";
+  system "mkdir", "run-main-tmp/";
+  my @restoreCommands;
+  for my $debugClassFile(glob "debug/*"){
+    if($debugClassFile =~ /^(?:.*\/)?([^\/]+)\.(\w+)\.java$/){
+      my ($pkg, $class) = ($1, $2);
+      my $targetDir = $pkg;
+      $targetDir =~ s/\./\//g;
+      $targetDir = "src/$targetDir";
+      system "cp", "$targetDir/$class.java", "run-main-tmp/$pkg\.$class.java";
+      push @restoreCommands, ["mv", "run-main-tmp/$pkg\.$class.java", "$targetDir/$class.java"];
+      system "cp", $debugClassFile, "$targetDir/$class.java";
+    }else{
+      die "ERROR: malformed debug class $debugClassFile\n";
+    }
+  }
+  system "javac", map {"src/$_"} @classes;
+  if($? != 0){
+    die "java failed\n";
+  }
+  system "java",
+    "-cp", "src",
+    "org.coolreader.crengine.WordTimingAudiobookMatcher",
+    @_;
+
+  for my $cmd(@restoreCommands){
+    system @$cmd;
+  }
+  system "rm", "-rf", "run-main-tmp/";
+  system "find", "src/", "-name", "*.class", "-delete";
+}
+
+&main(@ARGV);

android/src/org/coolreader/crengine/BaseActivity.java

@@ -1898,6 +1898,8 @@ public Properties loadSettings(BaseActivity activity, File file) {
 			// By default enable workaround to disable processing of abbreviations at the end of a sentence when using "Google Speech Services".
 			props.applyDefault(ReaderView.PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR, "1");
 
+			props.applyDefault(ReaderView.PROP_APP_TTS_USE_AUDIOBOOK, "1");
+
 			props.applyDefault(ReaderView.PROP_APP_THEME, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST1" : "LIGHT");
 			props.applyDefault(ReaderView.PROP_APP_THEME_DAY, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST1" : "LIGHT");
 			props.applyDefault(ReaderView.PROP_APP_THEME_NIGHT, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST2" : "DARK");

android/src/org/coolreader/crengine/DocView.java

@@ -27,6 +27,8 @@
 import java.io.IOException;
 import java.io.InputStream;
 
+import java.util.List;
+
 public class DocView {
 
 	public static final Logger log = L.create("dv");
@@ -281,6 +283,14 @@ public Bookmark getCurrentPageBookmarkNoRender() {
 		}
 	}
 
+	public List<SentenceInfo> getAllSentences() {
+		List<SentenceInfo> sentences;
+		synchronized(mutex) {
+			sentences = getAllSentencesInternal();
+		}
+		return sentences;
+	}
+
 	/**
 	 * Check whether document is formatted/rendered.
 	 * @return true if document is rendered, and e.g. retrieving of page image will not cause long activity (formatting etc.)
@@ -470,6 +480,8 @@ private native boolean applySettingsInternal(
 
 	private native Bookmark getCurrentPageBookmarkInternal();
 
+	private native List<SentenceInfo> getAllSentencesInternal();
+
 	private native boolean goToPositionInternal(String xPath, boolean saveToHistory);
 
 	private native PositionProperties getPositionPropsInternal(String xPath, boolean precise);

android/src/org/coolreader/crengine/InitAudiobookWordTimingsCallback.java

@@ -0,0 +1,5 @@
+package org.coolreader.crengine;
+
+public interface InitAudiobookWordTimingsCallback {
+	public void onComplete();
+}

android/src/org/coolreader/crengine/OptionsDialog.java

@@ -2655,6 +2655,12 @@ public void onTimedOut() {
 			mOptionsTTS.add(mTTSVoiceOption);
 		}
 		mOptionsTTS.add(new BoolOption(this, getString(R.string.options_tts_google_abbr_workaround), PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR).setComment(getString(R.string.options_tts_google_abbr_workaround_comment)).setDefaultValue("1").noIcon());
+		mOptionsTTS.add(
+			new BoolOption(
+				this,
+				getString(R.string.options_tts_use_audiobook),
+				PROP_APP_TTS_USE_AUDIOBOOK
+			).setDefaultValue("1").noIcon());
 		if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ECLAIR)
 			mOptionsTTS.add(new ListOption(this, getString(R.string.options_app_tts_stop_motion_timeout), PROP_APP_MOTION_TIMEOUT).add(mMotionTimeouts, mMotionTimeoutsTitles).setDefaultValue(Integer.toString(mMotionTimeouts[0])).noIcon());
 		mOptionsTTS.refresh();

android/src/org/coolreader/crengine/ReaderView.java

@@ -2420,6 +2420,7 @@ public void onCommand(final ReaderCommand cmd, final int param, final Runnable o
 					ttsToolbar = TTSToolbarDlg.showDialog(mActivity, ReaderView.this, ttsacc);
 					ttsToolbar.setOnCloseListener(() -> ttsToolbar = null);
 					ttsToolbar.setAppSettings(mSettings, null);
+					ttsToolbar.initAudiobookWordTimings(null);
 				}));
 			}
 			break;
@@ -3253,6 +3254,10 @@ public BookInfo getBookInfo() {
 		return mBookInfo;
 	}
 
+	public List<SentenceInfo> getAllSentences() {
+		return doc.getAllSentences();
+	}
+
 	private int mBatteryState = BATTERY_STATE_DISCHARGING;
 	private int mBatteryChargingConn = BATTERY_CHARGER_NO;
 	private int mBatteryChargeLevel = 0;

android/src/org/coolreader/crengine/SentenceInfo.java

@@ -0,0 +1,25 @@
+package org.coolreader.crengine;
+
+import java.io.File;
+import java.util.List;
+
+public class SentenceInfo {
+	public String text;
+	public String startPos;
+
+	public double startTime;
+	public boolean isFirstSentenceInAudioFile = false;
+	public File audioFile;
+	public List<String> words;
+	public SentenceInfo nextSentence;
+
+	public SentenceInfo() {
+	}
+
+	public void setStartPos(String startPos){
+		this.startPos = startPos;
+	}
+	public void setText(String text){
+		this.text = text;
+	}
+}

android/src/org/coolreader/crengine/SentenceInfoCache.java

@@ -0,0 +1,80 @@
+package org.coolreader.crengine;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SentenceInfoCache {
+	public static final Logger log = L.create("sentenceinfocache");
+
+	private final File sentenceInfoCacheFile;
+
+	public static List<SentenceInfo> maybeReadCache(File sentenceInfoCacheFile) {
+		try{
+			return new SentenceInfoCache(sentenceInfoCacheFile).readCache();
+		}catch(Exception e){
+			log.e("ERROR: could not read sentence info cache file: " + sentenceInfoCacheFile + " " + e);
+			return null;
+		}
+	}
+	public static void maybeWriteCache(File sentenceInfoCacheFile, List<SentenceInfo> allSentences) {
+		try{
+			new SentenceInfoCache(sentenceInfoCacheFile).writeCache(allSentences);
+		}catch(Exception e){
+			log.e("ERROR: could not write sentence info cache file: " + sentenceInfoCacheFile + " " + e);
+		}
+	}
+
+
+	public SentenceInfoCache(File sentenceInfoCacheFile) {
+		this.sentenceInfoCacheFile = sentenceInfoCacheFile;
+	}
+
+	public List<SentenceInfo> readCache() throws IOException {
+		List<SentenceInfo> allSentences = new ArrayList<>();
+		try (
+			BufferedReader br = new BufferedReader(new FileReader(sentenceInfoCacheFile));
+		) {
+			String line;
+			while ((line = br.readLine()) != null) {
+				SentenceInfo sentenceInfo = parseSentenceInfoLine(line);
+				if(sentenceInfo == null){
+					log.e("ERROR: could not parse sentence info cache line: " + line);
+					return null;
+				}
+				allSentences.add(sentenceInfo);
+			}
+		}
+		if(allSentences.isEmpty()){
+			return null;
+		}
+		return allSentences;
+	}
+
+	public void writeCache(List<SentenceInfo> allSentences) throws IOException {
+		FileWriter fw = new FileWriter(sentenceInfoCacheFile);
+		for(SentenceInfo s : allSentences){
+			fw.write(formatSentenceInfo(s));
+		}
+		fw.close();
+	}
+
+	private SentenceInfo parseSentenceInfoLine(String line){
+		int sep = line.indexOf(',');
+		if(sep < 0 || sep >= line.length()){
+			return null;
+		}
+		SentenceInfo sentenceInfo = new SentenceInfo();
+		sentenceInfo.startPos = line.substring(0, sep);
+		sentenceInfo.text = line.substring(sep+1);
+		return sentenceInfo;
+	}
+
+	private String formatSentenceInfo(SentenceInfo sentenceInfo){
+		return sentenceInfo.startPos + "," + sentenceInfo.text + "\n";
+	}
+}