From f90fe671ffed4bc525fcc79ff6fca2d8b0fd08b4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 24 Aug 2016 02:17:13 -0400 Subject: [PATCH] Synthesis with phonemes Changes the filesystem structure a bit, introduces new wave file reader and dictionaries, etc. --- Cargo.lock | 2 +- Cargo.toml | 2 +- config.toml | 4 + src/arpabet.rs | 8 +- src/audiobank.rs | 105 ++++++++++++++++++ src/config.rs | 34 ++++++ src/dictionary.rs | 6 +- src/error.rs | 27 +++-- src/handlers/audio_synth_handler.rs | 68 ++++-------- src/main.rs | 38 ++++--- src/synthesizer.rs | 158 ++++++++++++++++++++++++++++ 11 files changed, 372 insertions(+), 80 deletions(-) create mode 100644 src/audiobank.rs create mode 100644 src/config.rs create mode 100644 src/synthesizer.rs diff --git a/Cargo.lock b/Cargo.lock index 62043c3de6..a98921e803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ [root] name = "trumpet" -version = "0.0.2" +version = "0.1.0" dependencies = [ "clap 1.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "hound 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index e92f78461a..3d4947a1a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "trumpet" - version = "0.0.2" + version = "0.1.0" authors = [ "Brandon Thomas ", "Brandon Thomas " ] diff --git a/config.toml b/config.toml index 22b66b89ec..2da8d2d55a 100644 --- a/config.toml +++ b/config.toml @@ -4,3 +4,7 @@ sound_path_development = "/home/bt/Dropbox/jungle.horse/sounds" sound_path_production = "./sounds" +# Complete path to the phoneme dictionary file +phoneme_dictionary_file_development = "./dictionary/cmudict-0.7b" +phoneme_dictionary_file_production = "./dictionary/cmudict-0.7b" + diff --git a/src/arpabet.rs b/src/arpabet.rs index f208713aac..8f96dec8e3 100644 --- a/src/arpabet.rs +++ b/src/arpabet.rs @@ -1,6 +1,6 @@ // Copyright (c) 2016 Brandon Thomas -use error::Error; +use error::SynthError; use std::collections::HashMap; use regex::Regex; use std::fs::File; @@ -21,7 +21,7 @@ pub struct ArpabetDictionary { impl ArpabetDictionary { /// Read CMU's Arpabet Dictionary. /// See http://www.speech.cs.cmu.edu/cgi-bin/cmudict - pub fn load_from_file(filename: &str) -> Result { + pub fn load_from_file(filename: &str) -> Result { let mut f = try!(File::open(filename)); let mut reader = BufReader::new(f); @@ -52,14 +52,14 @@ impl ArpabetDictionary { } if map.is_empty() { - Err(Error::EmptyFile) + Err(SynthError::EmptyFile) } else { Ok(ArpabetDictionary { dictionary: map }) } } /// Get a polyphone from the dictionary. - pub fn get(&self, word: &str) -> Option { + pub fn get_polyphone(&self, word: &str) -> Option { self.dictionary.get(word).and_then(|p| { Some(p.iter().map(|s| s.to_string()).collect::>()) }) diff --git a/src/audiobank.rs b/src/audiobank.rs new file mode 100644 index 0000000000..a07b7b0f71 --- /dev/null +++ b/src/audiobank.rs @@ -0,0 +1,105 @@ +// Copyright (c) 2016 Brandon Thomas + +use error::SynthError; +use hound::WavReader; +use hound::WavSpec; +use hound::WavWriter; +use std::path::Path; +use std::path::PathBuf; + +/** + * Fetch wav audio files from the audio bank. + * + * The file database has a well-defined structure: + * + * /sounds + * /{speaker_name} + * /_words + * /all.wav + * /words.wav + * /_phonemes + * /IY0.wav + * /IY0.wav + * + * TODO: Caching of wav files in memory. + * TODO: Path generation is _insecure_. + */ +pub struct Audiobank { + /// Root path to the files. + audio_path: PathBuf, +} + +impl Audiobank { + /// CTOR. + pub fn new(audio_path: &str) -> Audiobank { + Audiobank { + audio_path: Path::new(audio_path).to_path_buf(), + } + } + + /** + * Get the wav data for the (speaker,word), or None if it does not exist. + */ + pub fn get_word(&self, speaker: &str, word: &str) -> Option> { + + println!("Audio Path: {:?}", self.audio_path); + + let path = self.audio_path.join(format!("{}/", speaker)) + .join("_words/") + .join(format!("{}.wav", word)); + + println!("Path: {:?}", path); + + let mut reader = match WavReader::open(path) { + Err(_) => { return None; }, + Ok(reader) => reader, + }; + + // TODO: Inefficient. + let mut all_samples = Vec::new(); + let samples = reader.samples::(); + for sample in samples { + all_samples.push(sample.unwrap()); + } + + Some(all_samples) + } + + /** + * Get the wav data for the (speaker,phoneme), or None if it does not exist. + */ + pub fn get_phoneme(&self, speaker: &str, phoneme: &str) -> Option> { + let path = self.audio_path.join(format!("{}/", speaker)) + .join("_phonemes/") + .join(format!("{}.wav", phoneme)); + + println!("Path: {:?}", path); + + let mut reader = match WavReader::open(path) { + Err(_) => { return None; }, + Ok(reader) => reader, + }; + + // TODO: Inefficient. + let mut all_samples = Vec::new(); + let samples = reader.samples::(); + for sample in samples { + all_samples.push(sample.unwrap()); + } + + Some(all_samples) + } + + // TODO: This should be removed. We should cache the wav headers. + pub fn get_spec(&self, speaker: &str, word: &str) -> Result { + let path = self.audio_path.join(format!("{}/", speaker)) + .join("_words/") + .join(format!("{}.wav", word)); + + println!("Path: {:?}", path); + + let reader = try!(WavReader::open(path)); + Ok(reader.spec()) + } +} + diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000000..99a6c687c3 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,34 @@ +// Copyright (c) 2016 Brandon Thomas + +use std::fs::File; +use std::io::Error; +use std::io::prelude::*; +use toml; + +#[derive(Debug, RustcEncodable, RustcDecodable)] +pub struct Config { + pub sound_path_development: String, + pub sound_path_production: String, + pub phoneme_dictionary_file_development: String, + pub phoneme_dictionary_file_production: String, +} + +impl Config { + pub fn read(filename: &str) -> Option { + match Config::read_file(filename) { + Err(_) => None, + Ok(contents) => { + let value = toml::Parser::new(&contents).parse().unwrap(); + toml::decode(toml::Value::Table(value)) + } + } + } + + fn read_file(filename: &str) -> Result { + let mut file = try!(File::open(filename)); + let mut buf = String::new(); + try!(file.read_to_string(&mut buf)); + Ok(buf) + } +} + diff --git a/src/dictionary.rs b/src/dictionary.rs index 9c43da6df5..a4d943d473 100644 --- a/src/dictionary.rs +++ b/src/dictionary.rs @@ -43,7 +43,7 @@ impl VocabularyLibrary { } let dirname = try!(get_filename(&entry)); - if ignorable_file(&dirname) { + if ignorable_file(&dirname) { continue; } @@ -64,7 +64,7 @@ impl VocabularyLibrary { impl Vocabulary { /** Read in the vocabulary from a directory. */ - pub fn read_from_directory(directory: &Path) -> + pub fn read_from_directory(directory: &Path) -> Result { let paths = try!(fs::read_dir(directory)); @@ -81,7 +81,7 @@ impl Vocabulary { let filename = try!(get_filename(&entry)); let word = filename.replace(".wav", ""); - if ignorable_file(&word) { + if ignorable_file(&word) { continue; } diff --git a/src/error.rs b/src/error.rs index dd467a1557..f8c1894724 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,22 +1,35 @@ // Copyright (c) 2016 Brandon Thomas -use std::error; -use std::io; -use std::fmt; +use hound::Error as HoundError; use std::convert::From; +use std::error; +use std::fmt; +use std::io; +/** The common error type used throughout the synthesizer. */ #[derive(Debug)] -pub enum Error { +pub enum SynthError { + /// Bad user input. + BadInput { description: &'static str }, + /// The file was empty or did not have parsable contents. EmptyFile, /// Wraps an IoError. IoError { cause: io::Error }, + + /// Wraps a Hound audio library error. + AudioError { cause: HoundError }, } -impl From for Error { - fn from(error: io::Error) -> Error { - Error::IoError { cause: error } +impl From for SynthError { + fn from(error: io::Error) -> SynthError { + SynthError::IoError { cause: error } } } +impl From for SynthError { + fn from(error: HoundError) -> SynthError { + SynthError::AudioError { cause: error } + } +} diff --git a/src/handlers/audio_synth_handler.rs b/src/handlers/audio_synth_handler.rs index 38af159426..dd7046a0a7 100644 --- a/src/handlers/audio_synth_handler.rs +++ b/src/handlers/audio_synth_handler.rs @@ -22,7 +22,10 @@ use std::io::Cursor; use std::io::Read; use std::io; use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::RwLock; use super::error_filter::build_error; +use synthesizer::Synthesizer; use urlencoded::UrlEncodedQuery; use words::split_sentence; @@ -52,6 +55,9 @@ enum SpeakerRequestError { /// Synthesizes audio from input. pub struct AudioSynthHandler { + /// The TTS synthesizer. + synthesizer: Arc>, + /// Root of where files can be served from. /// A PathBuf since `Path` can only be created as a borrow. directory: PathBuf, @@ -163,63 +169,31 @@ impl Handler for AudioSynthHandler { } impl AudioSynthHandler { - pub fn new(directory: &str) -> AudioSynthHandler { - AudioSynthHandler { directory: Path::new(directory).to_path_buf() } + pub fn new(synthesizer: Arc>, directory: &str) -> AudioSynthHandler { + AudioSynthHandler { + synthesizer: synthesizer, + directory: Path::new(directory).to_path_buf(), + } } // TODO: Return errors. /// Create audio from the sentence. fn create_audio(&self, speaker: &str, sentence: &str, volume: Option) -> Vec { - let mut words = split_sentence(sentence); - - if words.len() == 0 { - // TODO: Raise error! - } - - // This file adds extra silent padding at both ends. - words.insert(0, "_blank".to_string()); - words.push("_blank".to_string()); - - // Note: Keeping a list of buffered file readers is stupid and is simply - // being done for this example. I'll create a multithreaded shared LRU cache - // that reads from the disk and uses the dictionary word as the lookup key. - let mut file_readers : Vec>> = Vec::new(); - - // TODO: Cleanup. - let speaker_path = match self.get_speaker_path(speaker) { - Err(_) => { Path::new("").to_path_buf() }, // TODO: meh - Ok(p) => p, - }; - - for word in words.iter() { - let filename = self.get_file_path(speaker_path.as_path(), word); - let reader = WavReader::open(filename).unwrap(); - file_readers.push(reader); - } - - let mut all_samples : Vec = Vec::new(); - - for mut reader in file_readers { - let samples = reader.samples::(); - for sample in samples { - all_samples.push(sample.unwrap()); + match self.synthesizer.read() { + Err(_) => Vec::new(), // TODO Actual error. + Ok(synth) => { + match synth.generate(sentence, speaker, volume) { + Err(e) => { + println!("Error synthesizing: {:?}", e); + Vec::new() // TODO FIXME + }, + Ok(wav) => wav, + } } } - - // FIXME: Super inefficient. - let mut pcm_data : Vec = Vec::with_capacity(all_samples.len()); - for x in &all_samples { - let data = raise_volume(*x, volume); - pcm_data.push(data); - } - - let spec = self.get_spec(speaker_path.as_path(), &words[0]); - - self.write_buffer(&spec, pcm_data) } - fn sha_digest(&self, speaker: &str, sentence: &str, volume: Option) -> String { let mut hasher = Sha1::new(); hasher.input_str(speaker); diff --git a/src/main.rs b/src/main.rs index ac15800bf2..468c62f1e4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -15,15 +15,17 @@ extern crate toml; extern crate urlencoded; pub mod arpabet; -pub mod atom; -pub mod error; +pub mod audiobank; pub mod config; pub mod dictionary; +pub mod error; pub mod handlers; pub mod logger; +pub mod synthesizer; pub mod words; use arpabet::ArpabetDictionary; +use audiobank::Audiobank; use clap::{App, Arg, ArgMatches}; use config::Config; use dictionary::VocabularyLibrary; @@ -36,6 +38,9 @@ use logger::SimpleLogger; use resolve::hostname; use router::Router; use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::RwLock; +use synthesizer::Synthesizer; fn main() { SimpleLogger::init().unwrap(); @@ -59,9 +64,9 @@ fn main() { get_hostname(); - read_arpabet(); + let synthesizer = create_synthesizer(&config); - start_server(&config, port); + start_server(&config, port, synthesizer); } fn get_port(matches: &ArgMatches, default_port: u16) -> u16 { @@ -76,14 +81,16 @@ fn get_port(matches: &ArgMatches, default_port: u16) -> u16 { } } -fn start_server(config: &Config, port: u16) { +fn start_server(config: &Config, port: u16, synthesizer: Synthesizer) { let audio_path = &config.sound_path_development; let file_path = "./web"; let index = "index.html"; + let async_synth = Arc::new(RwLock::new(synthesizer)); + // TODO: Cross-cutting filter installation let mut router = Router::new(); - let mut chain = Chain::new(AudioSynthHandler::new(audio_path)); + let mut chain = Chain::new(AudioSynthHandler::new(async_synth.clone(), audio_path)); chain.link_after(ErrorFilter); router.get("/speak", chain); router.get("/words", VocabListHandler::new(audio_path)); @@ -103,20 +110,17 @@ fn get_hostname() { Ok(s) => { println!("Hostname: {}", s); }, Err(_) => {}, }; - - /*let len = 34u; - let mut buf = std::vec::from_elem(len, 0u8); - - let err = unsafe {gethostname (vec::raw::to_mut_ptr(buf) as *mut i8, len as u64)}; - if err != 0 { println("oops, gethostname failed"); return; }*/ } -fn read_arpabet() { - // TODO: Configurable path. - let dict = ArpabetDictionary::load_from_file("./dictionary/cmudict-0.7b").unwrap(); +fn create_synthesizer(config: &Config) -> Synthesizer { + info!("Reading Arpabet Dictionary..."); - let result = dict.get("nintendo"); + let arpabet_dictionary = ArpabetDictionary::load_from_file( + &config.phoneme_dictionary_file_development).unwrap(); - println!("Result: {:?}", result); + let audiobank = Audiobank::new(&config.sound_path_development); + + info!("Building Synthesizer..."); + Synthesizer::new(arpabet_dictionary, audiobank) } diff --git a/src/synthesizer.rs b/src/synthesizer.rs new file mode 100644 index 0000000000..97fb77aca9 --- /dev/null +++ b/src/synthesizer.rs @@ -0,0 +1,158 @@ +// Copyright (c) 2016 Brandon Thomas + +use arpabet::ArpabetDictionary; +use audiobank::Audiobank; +use dictionary::Vocabulary; +use dictionary::VocabularyLibrary; +use error::SynthError; +use hound::WavReader; +use hound::WavSpec; +use hound::WavWriter; +use std::error::Error; +use std::fs::File; +use std::fs; +use std::i16; +use std::io::BufReader; +use std::io::BufWriter; +use std::io::Cursor; +use std::io::Read; +use std::io; +use std::path::Path; +use std::path::PathBuf; +use words::split_sentence; + +pub type WavBytes = Vec; + +/** + * Generates wave files from text input. + */ +pub struct Synthesizer { + // TODO: For now we rely on word files existing on the filesystem + // at runtime. This is pretty lame. + ///// Word dictionary + //word_dictionary: Vocabulary, + + /// Arpabet dictionary, (Word) -> (PhonemeList) + arpabet_dictionary: ArpabetDictionary, + + /// WAV Audiobank for sound generation. + audiobank: Audiobank, +} + +impl Synthesizer { + /// CTOR + pub fn new(arpabet_dictionary: ArpabetDictionary, audiobank: Audiobank) -> Synthesizer { + Synthesizer { + arpabet_dictionary: arpabet_dictionary, + audiobank: audiobank, + } + } + + /** + * Generate a spoken wav from text input. + */ + pub fn generate(&self, sentence: &str, speaker: &str, + volume: Option) -> Result { + + let mut words = split_sentence(sentence); + + if words.len() == 0 { + return Err(SynthError::BadInput { + description: "There were no words to synthesize." + }); + } + + // This file adds extra silent padding at both ends. + words.insert(0, "_blank".to_string()); + words.push("_blank".to_string()); + + let mut concatenated_waveform : Vec = Vec::new(); + + // FIXME: Flight of the seagulls here. + for word in words.iter() { + match self.audiobank.get_word(speaker, word) { + Some(waveform_data) => { + // The word exists in the database. + concatenated_waveform.extend(waveform_data); + }, + None => { + // The word does not exist in the database, so use phonemes instead. + match self.arpabet_dictionary.get_polyphone(word) { + None => { + // TODO: ERROR (maybe add static based on word length!) + continue; + }, + Some(polyphone) => { + for phoneme in polyphone { + match self.audiobank.get_phoneme(speaker, &phoneme) { + None => { + // TODO: ERROR (maybe add static based on word length!) + continue; + }, + Some(waveform_data) => { + concatenated_waveform.extend(waveform_data); + }, + } + } + }, + } + }, + } + } + + // Adjust the volume of the waveform. + // FIXME: Super inefficient. + let mut pcm_data : Vec = Vec::with_capacity(concatenated_waveform.len()); + for x in &concatenated_waveform { + let data = raise_volume(*x, volume); + pcm_data.push(data); + } + + // TODO: Cache waveform headers. + let spec = try!(self.audiobank.get_spec(speaker, &words[0])); + + Ok(self.write_buffer(&spec, pcm_data)) + } + + fn write_buffer(&self, spec: &WavSpec, samples: Vec) -> Vec { + let mut bytes : Vec = Vec::new(); + let mut seek : Cursor> = Cursor::new(bytes); + let mut buffer = BufWriter::new(seek); + + { + let mut writer = WavWriter::new(&mut buffer, *spec); + for s in samples { + writer.write_sample(s).unwrap(); + } + writer.finalize().unwrap(); // TODO: Error + } + + match buffer.into_inner() { + Err(_) => { Vec::new() }, // TODO: Error + Ok(r) => { r.get_ref().to_vec() }, + } + } +} + +/// Raise the volume of a sample by changing its amplitude. +fn raise_volume(data: i16, volume: Option) -> i16 { + // TODO: Cleanup, make more efficient. + match volume { + None => data, + Some(vol) => { + let f : f32 = data as f32 * vol; + let g = f as i32; + + let h : i16 = if g > i16::MAX as i32 { + i16::MAX + } else if g < i16::MIN as i32 { + i16::MIN + } else { + g as i16 + }; + + h + }, + } +} +