Synthesis with phonemes

Changes the filesystem structure a bit, introduces new wave file
reader and dictionaries, etc.

Cargo.lock

@@ -1,6 +1,6 @@
 [root]
 name = "trumpet"
-version = "0.0.2"
+version = "0.1.0"
 dependencies = [
  "clap 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "hound 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
   name = "trumpet"
-  version = "0.0.2"
+  version = "0.1.0"
   authors = [ "Brandon Thomas <bt@brand.io>", 
               "Brandon Thomas <echelon@gmail.com>" ]
 

config.toml

@@ -4,3 +4,7 @@
 sound_path_development = "/home/bt/Dropbox/jungle.horse/sounds"
 sound_path_production = "./sounds"
 
+# Complete path to the phoneme dictionary file
+phoneme_dictionary_file_development = "./dictionary/cmudict-0.7b"
+phoneme_dictionary_file_production = "./dictionary/cmudict-0.7b"
+

src/arpabet.rs

@@ -1,6 +1,6 @@
 // Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
 
-use error::Error;
+use error::SynthError;
 use std::collections::HashMap;
 use regex::Regex;
 use std::fs::File;
@@ -21,7 +21,7 @@ pub struct ArpabetDictionary {
 impl ArpabetDictionary {
   /// Read CMU's Arpabet Dictionary.
   /// See http://www.speech.cs.cmu.edu/cgi-bin/cmudict
-  pub fn load_from_file(filename: &str) -> Result<ArpabetDictionary, Error> {
+  pub fn load_from_file(filename: &str) -> Result<ArpabetDictionary, SynthError> {
     let mut f = try!(File::open(filename));
     let mut reader = BufReader::new(f);
 
@@ -52,14 +52,14 @@ impl ArpabetDictionary {
     }
 
     if map.is_empty() {
-      Err(Error::EmptyFile)
+      Err(SynthError::EmptyFile)
     } else {
       Ok(ArpabetDictionary { dictionary: map })
     }
   }
 
   /// Get a polyphone from the dictionary.
-  pub fn get(&self, word: &str) -> Option<Polyphone> {
+  pub fn get_polyphone(&self, word: &str) -> Option<Polyphone> {
     self.dictionary.get(word).and_then(|p| {
       Some(p.iter().map(|s| s.to_string()).collect::<Vec<String>>())
     })

src/audiobank.rs

@@ -0,0 +1,105 @@
+// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
+
+use error::SynthError;
+use hound::WavReader;
+use hound::WavSpec;
+use hound::WavWriter;
+use std::path::Path;
+use std::path::PathBuf;
+
+/** 
+ * Fetch wav audio files from the audio bank.
+ *
+ * The file database has a well-defined structure:
+ *
+ *   /sounds
+ *     /{speaker_name}
+ *       /_words
+ *         /all.wav
+ *         /words.wav
+ *       /_phonemes
+ *         /IY0.wav
+ *         /IY0.wav
+ *
+ * TODO: Caching of wav files in memory.
+ * TODO: Path generation is _insecure_.
+ */
+pub struct Audiobank {
+  /// Root path to the files.
+  audio_path: PathBuf,
+}
+
+impl Audiobank {
+  /// CTOR.
+  pub fn new(audio_path: &str) -> Audiobank {
+    Audiobank {
+      audio_path: Path::new(audio_path).to_path_buf(),
+    }
+  }
+
+  /**
+   * Get the wav data for the (speaker,word), or None if it does not exist.
+   */
+  pub fn get_word(&self, speaker: &str, word: &str) -> Option<Vec<i16>> {
+
+    println!("Audio Path: {:?}", self.audio_path);
+
+    let path = self.audio_path.join(format!("{}/", speaker))
+        .join("_words/")
+        .join(format!("{}.wav", word));
+
+    println!("Path: {:?}", path);
+
+    let mut reader = match WavReader::open(path) {
+      Err(_) => { return None; },
+      Ok(reader) => reader,
+    };
+
+    // TODO: Inefficient.
+    let mut all_samples = Vec::new();
+    let samples = reader.samples::<i16>();
+    for sample in samples {
+      all_samples.push(sample.unwrap());
+    }
+
+    Some(all_samples)
+  }
+
+  /**
+   * Get the wav data for the (speaker,phoneme), or None if it does not exist.
+   */
+  pub fn get_phoneme(&self, speaker: &str, phoneme: &str) -> Option<Vec<i16>> {
+    let path = self.audio_path.join(format!("{}/", speaker))
+        .join("_phonemes/")
+        .join(format!("{}.wav", phoneme));
+
+    println!("Path: {:?}", path);
+
+    let mut reader = match WavReader::open(path) {
+      Err(_) => { return None; },
+      Ok(reader) => reader,
+    };
+
+    // TODO: Inefficient.
+    let mut all_samples = Vec::new();
+    let samples = reader.samples::<i16>();
+    for sample in samples {
+      all_samples.push(sample.unwrap());
+    }
+
+    Some(all_samples)
+  }
+
+  // TODO: This should be removed. We should cache the wav headers.
+  pub fn get_spec(&self, speaker: &str, word: &str) -> Result<WavSpec, SynthError> {
+    let path = self.audio_path.join(format!("{}/", speaker))
+        .join("_words/")
+        .join(format!("{}.wav", word));
+
+    println!("Path: {:?}", path);
+
+    let reader = try!(WavReader::open(path));
+    Ok(reader.spec())
+  }
+}
+

src/config.rs

@@ -0,0 +1,34 @@
+// Copyright (c) 2016 Brandon Thomas <bt@brand.io>
+
+use std::fs::File;
+use std::io::Error;
+use std::io::prelude::*;
+use toml;
+
+#[derive(Debug, RustcEncodable, RustcDecodable)]
+pub struct Config {
+  pub sound_path_development: String,
+  pub sound_path_production: String,
+  pub phoneme_dictionary_file_development: String,
+  pub phoneme_dictionary_file_production: String,
+}
+
+impl Config {
+  pub fn read(filename: &str) -> Option<Config> {
+    match Config::read_file(filename) {
+      Err(_) => None,
+      Ok(contents) => {
+        let value = toml::Parser::new(&contents).parse().unwrap();
+        toml::decode(toml::Value::Table(value))
+      }
+    }
+  }
+
+  fn read_file(filename: &str) -> Result<String, Error> {
+    let mut file = try!(File::open(filename));
+    let mut buf = String::new();
+    try!(file.read_to_string(&mut buf));
+    Ok(buf)
+  }
+}
+

src/dictionary.rs

@@ -43,7 +43,7 @@ impl VocabularyLibrary {
       }
 
       let dirname = try!(get_filename(&entry));
-      if ignorable_file(&dirname) { 
+      if ignorable_file(&dirname) {
         continue;
       }
 
@@ -64,7 +64,7 @@ impl VocabularyLibrary {
 
 impl Vocabulary {
   /** Read in the vocabulary from a directory. */
-  pub fn read_from_directory(directory: &Path) -> 
+  pub fn read_from_directory(directory: &Path) ->
       Result<Vocabulary, io::Error> {
 
     let paths = try!(fs::read_dir(directory));
@@ -81,7 +81,7 @@ impl Vocabulary {
       let filename = try!(get_filename(&entry));
       let word = filename.replace(".wav", "");
 
-      if ignorable_file(&word) { 
+      if ignorable_file(&word) {
         continue;
       }
 

src/error.rs

@@ -1,22 +1,35 @@
 // Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
 
-use std::error;
-use std::io;
-use std::fmt;
+use hound::Error as HoundError;
 use std::convert::From;
+use std::error;
+use std::fmt;
+use std::io;
 
+/** The common error type used throughout the synthesizer. */
 #[derive(Debug)]
-pub enum Error {
+pub enum SynthError {
+  /// Bad user input.
+  BadInput { description: &'static str },
+
   /// The file was empty or did not have parsable contents.
   EmptyFile,
 
   /// Wraps an IoError.
   IoError { cause: io::Error },
+
+  /// Wraps a Hound audio library error.
+  AudioError { cause: HoundError },
 }
 
-impl From<io::Error> for Error {
-  fn from(error: io::Error) -> Error {
-    Error::IoError { cause: error }
+impl From<io::Error> for SynthError {
+  fn from(error: io::Error) -> SynthError {
+    SynthError::IoError { cause: error }
   }
 }
 
+impl From<HoundError> for SynthError {
+  fn from(error: HoundError) -> SynthError {
+    SynthError::AudioError { cause: error }
+  }
+}

src/handlers/audio_synth_handler.rs

@@ -22,7 +22,10 @@ use std::io::Cursor;
 use std::io::Read;
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::RwLock;
 use super::error_filter::build_error;
+use synthesizer::Synthesizer;
 use urlencoded::UrlEncodedQuery;
 use words::split_sentence;
 
@@ -52,6 +55,9 @@ enum SpeakerRequestError {
 
 /// Synthesizes audio from input.
 pub struct AudioSynthHandler {
+  /// The TTS synthesizer.
+  synthesizer: Arc<RwLock<Synthesizer>>,
+
   /// Root of where files can be served from.
   /// A PathBuf since `Path` can only be created as a borrow.
   directory: PathBuf,
@@ -163,63 +169,31 @@ impl Handler for AudioSynthHandler {
 }
 
 impl AudioSynthHandler {
-  pub fn new(directory: &str) -> AudioSynthHandler {
-    AudioSynthHandler { directory: Path::new(directory).to_path_buf() }
+  pub fn new(synthesizer: Arc<RwLock<Synthesizer>>, directory: &str) -> AudioSynthHandler {
+    AudioSynthHandler { 
+      synthesizer: synthesizer,
+      directory: Path::new(directory).to_path_buf(),
+    }
   }
 
   // TODO: Return errors.
   /// Create audio from the sentence.
   fn create_audio(&self, speaker: &str, sentence: &str, volume: Option<f32>)
       -> Vec<u8> {
-    let mut words = split_sentence(sentence);
-
-    if words.len() == 0 {
-      // TODO: Raise error!
-    }
-
-    // This file adds extra silent padding at both ends.
-    words.insert(0, "_blank".to_string());
-    words.push("_blank".to_string());
-
-    // Note: Keeping a list of buffered file readers is stupid and is simply 
-    // being done for this example. I'll create a multithreaded shared LRU cache
-    // that reads from the disk and uses the dictionary word as the lookup key.
-    let mut file_readers : Vec<WavReader<BufReader<File>>> = Vec::new();
-
-    // TODO: Cleanup.
-    let speaker_path = match self.get_speaker_path(speaker) {
-      Err(_) => { Path::new("").to_path_buf() }, // TODO: meh
-      Ok(p) => p,
-    };
-
-    for word in words.iter() {
-      let filename = self.get_file_path(speaker_path.as_path(), word);
-      let reader = WavReader::open(filename).unwrap();
-      file_readers.push(reader);
-    }
-
-    let mut all_samples : Vec<i16> = Vec::new();
-
-    for mut reader in file_readers {
-      let samples = reader.samples::<i16>();
-      for sample in samples {
-        all_samples.push(sample.unwrap());
+    match self.synthesizer.read() {
+      Err(_) => Vec::new(), // TODO Actual error.
+      Ok(synth) => {
+        match synth.generate(sentence, speaker, volume) {
+          Err(e) => {
+            println!("Error synthesizing: {:?}", e);
+            Vec::new() // TODO FIXME
+          },
+          Ok(wav) => wav,
+        }
       }
     }
-
-    // FIXME: Super inefficient.
-    let mut pcm_data : Vec<i16> = Vec::with_capacity(all_samples.len());
-    for x in &all_samples {
-      let data = raise_volume(*x, volume);
-      pcm_data.push(data);
-    }
-
-    let spec = self.get_spec(speaker_path.as_path(), &words[0]);
-
-    self.write_buffer(&spec, pcm_data)
   }
 
-
   fn sha_digest(&self, speaker: &str, sentence: &str, volume: Option<f32>) -> String {
     let mut hasher = Sha1::new();
     hasher.input_str(speaker);

src/main.rs

@@ -15,15 +15,17 @@ extern crate toml;
 extern crate urlencoded;
 
 pub mod arpabet;
-pub mod atom;
-pub mod error;
+pub mod audiobank;
 pub mod config;
 pub mod dictionary;
+pub mod error;
 pub mod handlers;
 pub mod logger;
+pub mod synthesizer;
 pub mod words;
 
 use arpabet::ArpabetDictionary;
+use audiobank::Audiobank;
 use clap::{App, Arg, ArgMatches};
 use config::Config;
 use dictionary::VocabularyLibrary;
@@ -36,6 +38,9 @@ use logger::SimpleLogger;
 use resolve::hostname;
 use router::Router;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::sync::RwLock;
+use synthesizer::Synthesizer;
 
 fn main() {
   SimpleLogger::init().unwrap();
@@ -59,9 +64,9 @@ fn main() {
 
   get_hostname();
 
-  read_arpabet();
+  let synthesizer = create_synthesizer(&config);
 
-  start_server(&config, port);
+  start_server(&config, port, synthesizer);
 }
 
 fn get_port(matches: &ArgMatches, default_port: u16) -> u16 {
@@ -76,14 +81,16 @@ fn get_port(matches: &ArgMatches, default_port: u16) -> u16 {
   }
 }
 
-fn start_server(config: &Config, port: u16) {
+fn start_server(config: &Config, port: u16, synthesizer: Synthesizer) {
   let audio_path = &config.sound_path_development;
   let file_path = "./web";
   let index = "index.html";
 
+  let async_synth = Arc::new(RwLock::new(synthesizer));
+
   // TODO: Cross-cutting filter installation
   let mut router = Router::new();
-  let mut chain = Chain::new(AudioSynthHandler::new(audio_path));
+  let mut chain = Chain::new(AudioSynthHandler::new(async_synth.clone(), audio_path));
   chain.link_after(ErrorFilter);
   router.get("/speak", chain);
   router.get("/words", VocabListHandler::new(audio_path));
@@ -103,20 +110,17 @@ fn get_hostname() {
     Ok(s) => { println!("Hostname: {}", s); },
     Err(_) => {},
   };
-
-  /*let len = 34u;
-  let mut buf = std::vec::from_elem(len, 0u8);
-
-  let err = unsafe {gethostname (vec::raw::to_mut_ptr(buf) as *mut i8, len as u64)};
-  if err != 0 { println("oops, gethostname failed"); return; }*/
 }
 
-fn read_arpabet() {
-  // TODO: Configurable path.
-  let dict = ArpabetDictionary::load_from_file("./dictionary/cmudict-0.7b").unwrap();
+fn create_synthesizer(config: &Config) -> Synthesizer {
+  info!("Reading Arpabet Dictionary...");
 
-  let result = dict.get("nintendo");
+  let arpabet_dictionary = ArpabetDictionary::load_from_file(
+      &config.phoneme_dictionary_file_development).unwrap();
 
-  println!("Result: {:?}", result);
+  let audiobank = Audiobank::new(&config.sound_path_development);
+
+  info!("Building Synthesizer...");
+  Synthesizer::new(arpabet_dictionary, audiobank)
 }
 

src/synthesizer.rs

@@ -0,0 +1,158 @@
+// Copyright (c) 2016 Brandon Thomas <bt@brand.io, echelon@gmail.com>
+
+use arpabet::ArpabetDictionary;
+use audiobank::Audiobank;
+use dictionary::Vocabulary;
+use dictionary::VocabularyLibrary;
+use error::SynthError;
+use hound::WavReader;
+use hound::WavSpec;
+use hound::WavWriter;
+use std::error::Error;
+use std::fs::File;
+use std::fs;
+use std::i16;
+use std::io::BufReader;
+use std::io::BufWriter;
+use std::io::Cursor;
+use std::io::Read;
+use std::io;
+use std::path::Path;
+use std::path::PathBuf;
+use words::split_sentence;
+
+pub type WavBytes = Vec<u8>;
+
+/**
+ * Generates wave files from text input.
+ */
+pub struct Synthesizer {
+  // TODO: For now we rely on word files existing on the filesystem
+  // at runtime. This is pretty lame.
+  ///// Word dictionary
+  //word_dictionary: Vocabulary,
+
+  /// Arpabet dictionary, (Word) -> (PhonemeList)
+  arpabet_dictionary: ArpabetDictionary,
+
+  /// WAV Audiobank for sound generation.
+  audiobank: Audiobank,
+}
+
+impl Synthesizer {
+  /// CTOR
+  pub fn new(arpabet_dictionary: ArpabetDictionary, audiobank: Audiobank) -> Synthesizer {
+    Synthesizer {
+      arpabet_dictionary: arpabet_dictionary,
+      audiobank: audiobank,
+    }
+  }
+
+  /**
+   * Generate a spoken wav from text input.
+   */
+  pub fn generate(&self, sentence: &str, speaker: &str, 
+                  volume: Option<f32>) -> Result<WavBytes, SynthError> {
+
+    let mut words = split_sentence(sentence);
+
+    if words.len() == 0 {
+      return Err(SynthError::BadInput { 
+        description: "There were no words to synthesize."
+      });
+    }
+
+    // This file adds extra silent padding at both ends.
+    words.insert(0, "_blank".to_string());
+    words.push("_blank".to_string());
+    
+    let mut concatenated_waveform : Vec<i16> = Vec::new();
+
+    // FIXME: Flight of the seagulls here.
+    for word in words.iter() {
+      match self.audiobank.get_word(speaker, word) {
+        Some(waveform_data) => {
+          // The word exists in the database.
+          concatenated_waveform.extend(waveform_data);
+        },
+        None => {
+          // The word does not exist in the database, so use phonemes instead.
+          match self.arpabet_dictionary.get_polyphone(word) {
+            None => {
+              // TODO: ERROR (maybe add static based on word length!)
+              continue;
+            },
+            Some(polyphone) => {
+              for phoneme in polyphone {
+                match self.audiobank.get_phoneme(speaker, &phoneme) {
+                  None => {
+                    // TODO: ERROR (maybe add static based on word length!)
+                    continue;
+                  },
+                  Some(waveform_data) => {
+                    concatenated_waveform.extend(waveform_data);
+                  },
+                }
+              }
+            },
+          }
+        },
+      }
+    }
+
+    // Adjust the volume of the waveform.
+    // FIXME: Super inefficient.
+    let mut pcm_data : Vec<i16> = Vec::with_capacity(concatenated_waveform.len());
+    for x in &concatenated_waveform {
+      let data = raise_volume(*x, volume);
+      pcm_data.push(data);
+    }
+
+    // TODO: Cache waveform headers.
+    let spec = try!(self.audiobank.get_spec(speaker, &words[0]));
+
+    Ok(self.write_buffer(&spec, pcm_data))
+  }
+
+  fn write_buffer(&self, spec: &WavSpec, samples: Vec<i16>) -> Vec<u8> {
+    let mut bytes : Vec<u8> = Vec::new();
+    let mut seek : Cursor<Vec<u8>> = Cursor::new(bytes);
+    let mut buffer = BufWriter::new(seek);
+
+    {
+      let mut writer = WavWriter::new(&mut buffer, *spec);
+      for s in samples {
+        writer.write_sample(s).unwrap();
+      }
+      writer.finalize().unwrap(); // TODO: Error
+    }
+
+    match buffer.into_inner() {
+      Err(_) => { Vec::new() }, // TODO: Error
+      Ok(r) => { r.get_ref().to_vec() },
+    }
+  }
+}
+
+/// Raise the volume of a sample by changing its amplitude.
+fn raise_volume(data: i16, volume: Option<f32>) -> i16 {
+  // TODO: Cleanup, make more efficient.
+  match volume {
+    None => data,
+    Some(vol) => {
+      let f : f32 = data as f32 * vol;
+      let g = f as i32;
+
+      let h : i16 = if g > i16::MAX as i32 {
+        i16::MAX
+      } else if g < i16::MIN as i32 {
+        i16::MIN
+      } else {
+        g as i16
+      };
+
+      h
+    },
+  }
+}
+