From f096ae07bf36d0f5df6ee36f3dfd701de41e886d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 21 Oct 2019 02:58:05 -0400 Subject: [PATCH] Changing proto schema. Degradation in quality. I had something really close. That might not have found it into a commit between now and the last commit :( --- cycle_gan/protos/audio_pb2.py | 128 ++++++++++++++++++++++++++++------ cycle_gan/sidecar.py | 45 +++++++----- protos/audio.proto | 30 +++++++- src/main.rs | 24 +++++-- src/protos/voder_audio.rs | 48 ++++++++++--- 5 files changed, 218 insertions(+), 57 deletions(-) diff --git a/cycle_gan/protos/audio_pb2.py b/cycle_gan/protos/audio_pb2.py index f77b585fe4..9df0295cd3 100644 --- a/cycle_gan/protos/audio_pb2.py +++ b/cycle_gan/protos/audio_pb2.py @@ -19,13 +19,99 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='protos/audio.proto', package='voder_audio', syntax='proto3', - serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xec\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x12\x1b\n\x13model_sampling_rate\x18\t \x01(\x05\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3') + serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xb7\x04\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x43\n\rvocode_params\x18\n \x01(\x0b\x32,.voder_audio.VocodeAudioRequest.VocodeParams\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x1a\xca\x02\n\x0cVocodeParams\x12\x1b\n\x13initial_sample_rate\x18\x01 \x01(\x05\x12\x1c\n\x14original_source_rate\x18\t \x01(\x05\x12\x1c\n\x14pre_convert_resample\x18\x02 \x01(\x08\x12!\n\x19pre_convert_resample_rate\x18\x03 \x01(\x05\x12&\n\x1epre_convert_resample_save_file\x18\x04 \x01(\x08\x12*\n\"model_hyperparameter_sampling_rate\x18\x05 \x01(\x05\x12\x1d\n\x15post_convert_resample\x18\x06 \x01(\x08\x12\"\n\x1apost_convert_resample_rate\x18\x07 \x01(\x05\x12\'\n\x1fpost_convert_resample_save_file\x18\x08 \x01(\x08\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3') ) _sym_db.RegisterFileDescriptor(DESCRIPTOR) +_VOCODEAUDIOREQUEST_VOCODEPARAMS = _descriptor.Descriptor( + name='VocodeParams', + full_name='voder_audio.VocodeAudioRequest.VocodeParams', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='initial_sample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.initial_sample_rate', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='original_source_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_rate', index=1, + number=9, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample', index=2, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_rate', index=3, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_save_file', index=4, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='model_hyperparameter_sampling_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_hyperparameter_sampling_rate', index=5, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample', index=6, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_rate', index=7, + number=7, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_save_file', index=8, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=273, + serialized_end=603, +) + _VOCODEAUDIOREQUEST = _descriptor.Descriptor( name='VocodeAudioRequest', full_name='voder_audio.VocodeAudioRequest', @@ -41,9 +127,9 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor( is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='sample_rate', full_name='voder_audio.VocodeAudioRequest.sample_rate', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, + name='vocode_params', full_name='voder_audio.VocodeAudioRequest.vocode_params', index=1, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), @@ -76,30 +162,16 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor( is_extension=False, extension_scope=None, options=None), _descriptor.FieldDescriptor( - name='output_rate', full_name='voder_audio.VocodeAudioRequest.output_rate', index=6, - number=7, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=7, + name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=6, number=8, type=8, cpp_type=7, label=1, has_default_value=False, default_value=False, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), - _descriptor.FieldDescriptor( - name='model_sampling_rate', full_name='voder_audio.VocodeAudioRequest.model_sampling_rate', index=8, - number=9, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), ], extensions=[ ], - nested_types=[], + nested_types=[_VOCODEAUDIOREQUEST_VOCODEPARAMS, ], enum_types=[ ], options=None, @@ -109,7 +181,7 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor( oneofs=[ ], serialized_start=36, - serialized_end=272, + serialized_end=603, ) @@ -139,19 +211,29 @@ _VOCODEAUDIORESPONSE = _descriptor.Descriptor( extension_ranges=[], oneofs=[ ], - serialized_start=274, - serialized_end=316, + serialized_start=605, + serialized_end=647, ) +_VOCODEAUDIOREQUEST_VOCODEPARAMS.containing_type = _VOCODEAUDIOREQUEST +_VOCODEAUDIOREQUEST.fields_by_name['vocode_params'].message_type = _VOCODEAUDIOREQUEST_VOCODEPARAMS DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST DESCRIPTOR.message_types_by_name['VocodeAudioResponse'] = _VOCODEAUDIORESPONSE VocodeAudioRequest = _reflection.GeneratedProtocolMessageType('VocodeAudioRequest', (_message.Message,), dict( + + VocodeParams = _reflection.GeneratedProtocolMessageType('VocodeParams', (_message.Message,), dict( + DESCRIPTOR = _VOCODEAUDIOREQUEST_VOCODEPARAMS, + __module__ = 'protos.audio_pb2' + # @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest.VocodeParams) + )) + , DESCRIPTOR = _VOCODEAUDIOREQUEST, __module__ = 'protos.audio_pb2' # @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest) )) _sym_db.RegisterMessage(VocodeAudioRequest) +_sym_db.RegisterMessage(VocodeAudioRequest.VocodeParams) VocodeAudioResponse = _reflection.GeneratedProtocolMessageType('VocodeAudioResponse', (_message.Message,), dict( DESCRIPTOR = _VOCODEAUDIORESPONSE, diff --git a/cycle_gan/sidecar.py b/cycle_gan/sidecar.py index 170f2dba73..efdb912147 100755 --- a/cycle_gan/sidecar.py +++ b/cycle_gan/sidecar.py @@ -62,7 +62,7 @@ class Converter(): frame_period = self.frame_period, multiple = 4) - librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate) + #librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate) f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, @@ -97,11 +97,11 @@ class Converter(): #librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) # TODO: Perhaps ditch this. It's probably unnecessary work. - upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) - pcm_data = upsampled.astype(np.float64) + #upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) + #pcm_data = upsampled.astype(np.float64) #stereo_pcm_data = np.tile(pcm_data, (2,1)).T #return stereo_pcm_data.astype(np.float32) - librosa.output.write_wav('model_output.wav', pcm_data, 48000) + #librosa.output.write_wav('model_output.wav', pcm_data, 48000) #return wav return wav_transformed @@ -130,8 +130,8 @@ def temp_file_name(suffix='.wav'): name = os.path.basename(temp_file.name) return os.path.join(TEMP_DIR.name, name) -def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False, - source_rate=44100, output_rate=44100, model_sampling_rate=16000): +def convert(audio, vocode_params=None, + skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False): #audio = np.array(audio, dtype=np.int16) #data, samplerate = soundfile.read(audio) #print('samplerate', samplerate) @@ -149,6 +149,14 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis 1) Data should be float64 2) Output is BYTES!! Not floats. 3) Result is still somehow mono!? + -------------------------------------------------- + + demo_server.py + + capture sample rate = 44100 + downsample for algorithm = 16000 + upsample after done = 48000 + """ audio = np.array(audio, dtype=np.float64) @@ -161,38 +169,43 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis if save_files: filename = temp_file_name('.wav') print('----- Original wav file out: {}'.format(filename)) - scipy.io.wavfile.write(filename, source_rate, audio) + scipy.io.wavfile.write(filename, vocode_params.original_source_rate, audio) # NB: Convert the input stereo signal into mono. # In the future the frontend should be responsible for sampling details. #audio = audio[:, 0] - if not skip_resample: - print("Resampling audio from {} Hz to 16000 Hz".format(source_rate)) - audio = librosa.resample(audio, source_rate, 16000) + if vocode_params.pre_convert_resample: + print("Resampling audio from {} Hz to {} Hz".format(vocode_params.original_source_rate, + vocode_params.pre_convert_resample_rate)) + audio = librosa.resample(audio, vocode_params.original_source_rate, vocode_params.pre_convert_resample_rate) print('resampled_audio.shape', audio.shape) print('resampled_audio.type', audio.dtype) if save_files: filename = temp_file_name('.wav') print('----- Downsampled file out: {}'.format(filename)) - scipy.io.wavfile.write(filename, 16000, audio) + scipy.io.wavfile.write(filename, vocode_params.pre_convert_resample_rate, audio) if skip_vocode: return audio - results = converter.convert_partial(audio, conversion_direction='A2B', model_sampling_rate=model_sampling_rate) + results = converter.convert_partial(audio, + conversion_direction='A2B', + model_sampling_rate=vocode_params.model_hyperparameter_sampling_rate) print('results.type', type(results)) print('results.len', len(results)) + """ if output_rate != 16000: - print("Resampling output audio from {} Hz to 16000 Hz".format(source_rate)) + print("Resampling output audio from {} Hz to 16000 Hz".format(vocode_params.source_rate)) #consume_rate = 68000 # Experimentally determined for Rust lib 'CPAL' results = librosa.resample(results, 16000, output_rate) if save_files: filename = temp_file_name('.wav') print('----- Upsampled (transformed) file out: {}'.format(filename)) scipy.io.wavfile.write(filename, output_rate, results) + """ if discard_vocoded_audio: return audio @@ -215,17 +228,15 @@ def main(): message = socket.recv() vocode_request = VocodeAudioRequest.FromString(message) + vocode_params = vocode_request.vocode_params queue.extend(vocode_request.float_audio) if len(queue) >= vocode_request.buffer_size_minimum: #results = queue[:] results = convert(queue, - source_rate=vocode_request.sample_rate, - output_rate=vocode_request.output_rate, - model_sampling_rate=vocode_request.model_sampling_rate, + vocode_params=vocode_params, skip_vocode=vocode_request.skip_vocode, - skip_resample=vocode_request.skip_resample, save_files=vocode_request.save_files, discard_vocoded_audio=vocode_request.discard_vocoded_audio) queue = [] diff --git a/protos/audio.proto b/protos/audio.proto index 334ed67380..6771ec1437 100644 --- a/protos/audio.proto +++ b/protos/audio.proto @@ -4,9 +4,33 @@ package voder_audio; message VocodeAudioRequest { repeated float float_audio = 1; + message VocodeParams { + // The initial sample rate coming from the microphone. + int32 initial_sample_rate = 1; + + // Original sample rate + int32 original_source_rate = 9; + + // Resample before passing to the algorithm? + bool pre_convert_resample = 2; + int32 pre_convert_resample_rate = 3; + bool pre_convert_resample_save_file = 4; + + // The hard-coded hyperparameter of the model. + // Probably don't want to change from 16000. + int32 model_hyperparameter_sampling_rate = 5; + + // Resample after passing to the algorithm? + bool post_convert_resample = 6; + int32 post_convert_resample_rate = 7; + bool post_convert_resample_save_file = 8; + } + + VocodeParams vocode_params = 10; + // We'll resend these params for every batch, but will only use // the last batch's settings. Kind of a dumb API, but it'll work. - int32 sample_rate = 2; + //int32 sample_rate = 2; bool skip_resample = 3; bool skip_vocode = 4; @@ -18,13 +42,13 @@ message VocodeAudioRequest { int32 buffer_size_minimum = 6; // Requested output rate - int32 output_rate = 7; + //int32 output_rate = 7; // Discard the vocoded audio and return the original bool discard_vocoded_audio = 8; // Rate param for the model - int32 model_sampling_rate = 9; + //int32 model_sampling_rate = 9; } message VocodeAudioResponse { diff --git a/src/main.rs b/src/main.rs index c158e095dc..e2a7768878 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,6 +41,7 @@ use std::process::exit; use std::sync::Arc; use std::thread; use zmq::{Error, Socket}; +use protos::voder_audio::vocode_audio_request::VocodeParams; fn main() { //print_version(); @@ -101,16 +102,31 @@ fn run_cpal_audio() -> Result<(), failure::Error> { }; let mut vocode_request = VocodeAudioRequest::default(); - vocode_request.sample_rate = 16000; - vocode_request.output_rate = 16000; - vocode_request.model_sampling_rate = 88000; + //vocode_request.sample_rate = 16000; + //vocode_request.output_rate = 16000; + //vocode_request.model_sampling_rate = 88000; // This is close! + //vocode_request.model_sampling_rate = 88000; vocode_request.skip_resample = false; vocode_request.skip_vocode = false; vocode_request.discard_vocoded_audio = false; vocode_request.save_files = false; - vocode_request.buffer_size_minimum = 100000; + //vocode_request.buffer_size_minimum = 5000; // AWFUL. SO CHOPPY. + vocode_request.buffer_size_minimum = 50000; // Practically real time, but lots more phase distortion. + //vocode_request.buffer_size_minimum = 70000; // Sounds pretty good. + //vocode_request.buffer_size_minimum = 100000; // This sounds good! A bit slow ~3seconds. + //vocode_request.buffer_size_minimum = 200000; vocode_request.float_audio = drained.clone(); + let mut vocode_params = VocodeParams::default(); + vocode_params.original_source_rate = 16000; + vocode_params.pre_convert_resample = true; + vocode_params.pre_convert_resample_rate = 16000; + vocode_params.model_hyperparameter_sampling_rate = 88000; + vocode_params.post_convert_resample = false; + vocode_params.post_convert_resample_rate = 16000; + + vocode_request.vocode_params = Some(vocode_params); + let mut encoded_bytes = Vec::with_capacity(vocode_request.encoded_len()); vocode_request.encode(&mut encoded_bytes).unwrap(); diff --git a/src/protos/voder_audio.rs b/src/protos/voder_audio.rs index ba993a3a16..74fd158119 100644 --- a/src/protos/voder_audio.rs +++ b/src/protos/voder_audio.rs @@ -2,10 +2,12 @@ pub struct VocodeAudioRequest { #[prost(float, repeated, tag="1")] pub float_audio: ::std::vec::Vec, - /// We'll resend these params for every batch, but will only use - /// the last batch's settings. Kind of a dumb API, but it'll work. - #[prost(int32, tag="2")] - pub sample_rate: i32, + #[prost(message, optional, tag="10")] + pub vocode_params: ::std::option::Option, + // We'll resend these params for every batch, but will only use + // the last batch's settings. Kind of a dumb API, but it'll work. + //int32 sample_rate = 2; + #[prost(bool, tag="3")] pub skip_resample: bool, #[prost(bool, tag="4")] @@ -16,15 +18,41 @@ pub struct VocodeAudioRequest { /// How big we let the buffer grow before running 'convert'. #[prost(int32, tag="6")] pub buffer_size_minimum: i32, - /// Requested output rate - #[prost(int32, tag="7")] - pub output_rate: i32, + // Requested output rate + //int32 output_rate = 7; + /// Discard the vocoded audio and return the original #[prost(bool, tag="8")] pub discard_vocoded_audio: bool, - /// Rate param for the model - #[prost(int32, tag="9")] - pub model_sampling_rate: i32, +} +pub mod vocode_audio_request { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct VocodeParams { + /// The initial sample rate coming from the microphone. + #[prost(int32, tag="1")] + pub initial_sample_rate: i32, + /// Original sample rate + #[prost(int32, tag="9")] + pub original_source_rate: i32, + /// Resample before passing to the algorithm? + #[prost(bool, tag="2")] + pub pre_convert_resample: bool, + #[prost(int32, tag="3")] + pub pre_convert_resample_rate: i32, + #[prost(bool, tag="4")] + pub pre_convert_resample_save_file: bool, + /// The hard-coded hyperparameter of the model. + /// Probably don't want to change from 16000. + #[prost(int32, tag="5")] + pub model_hyperparameter_sampling_rate: i32, + /// Resample after passing to the algorithm? + #[prost(bool, tag="6")] + pub post_convert_resample: bool, + #[prost(int32, tag="7")] + pub post_convert_resample_rate: i32, + #[prost(bool, tag="8")] + pub post_convert_resample_save_file: bool, + } } #[derive(Clone, PartialEq, ::prost::Message)] pub struct VocodeAudioResponse {