gsadmin/artcraft
1
0
Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-10-26 11:09:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Changing proto schema. Degradation in quality.

Browse Source 
I had something really close. That might not have found it into a
commit between now and the last commit :(
This commit is contained in:
Brandon Thomas
2019-10-21 02:58:05 -04:00
parent af20a57e8a
commit f096ae07bf
5 changed files with 218 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cycle_gan/protos/audio_pb2.py
+105 -23
View File
@@ -19,13 +19,99 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='protos/audio.proto',
package='voder_audio',
syntax='proto3',
serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xec\x01\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x13\n\x0bsample_rate\x18\x02 \x01(\x05\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x13\n\x0boutput_rate\x18\x07 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x12\x1b\n\x13model_sampling_rate\x18\t \x01(\x05\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3')
serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xb7\x04\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x43\n\rvocode_params\x18\n \x01(\x0b\x32,.voder_audio.VocodeAudioRequest.VocodeParams\x12\x15\n\rskip_resample\x18\x03 \x01(\x08\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x12\n\nsave_files\x18\x05 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x1a\xca\x02\n\x0cVocodeParams\x12\x1b\n\x13initial_sample_rate\x18\x01 \x01(\x05\x12\x1c\n\x14original_source_rate\x18\t \x01(\x05\x12\x1c\n\x14pre_convert_resample\x18\x02 \x01(\x08\x12!\n\x19pre_convert_resample_rate\x18\x03 \x01(\x05\x12&\n\x1epre_convert_resample_save_file\x18\x04 \x01(\x08\x12*\n\"model_hyperparameter_sampling_rate\x18\x05 \x01(\x05\x12\x1d\n\x15post_convert_resample\x18\x06 \x01(\x08\x12\"\n\x1apost_convert_resample_rate\x18\x07 \x01(\x05\x12\'\n\x1fpost_convert_resample_save_file\x18\x08 \x01(\x08\"*\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x62\x06proto3')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_VOCODEAUDIOREQUEST_VOCODEPARAMS = _descriptor.Descriptor(
name='VocodeParams',
full_name='voder_audio.VocodeAudioRequest.VocodeParams',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='initial_sample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.initial_sample_rate', index=0,
number=1, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='original_source_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_rate', index=1,
number=9, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='pre_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample', index=2,
number=2, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='pre_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_rate', index=3,
number=3, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='pre_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_save_file', index=4,
number=4, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='model_hyperparameter_sampling_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_hyperparameter_sampling_rate', index=5,
number=5, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='post_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample', index=6,
number=6, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='post_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_rate', index=7,
number=7, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='post_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_save_file', index=8,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=273,
serialized_end=603,
)
_VOCODEAUDIOREQUEST = _descriptor.Descriptor(
name='VocodeAudioRequest',
full_name='voder_audio.VocodeAudioRequest',
@@ -41,9 +127,9 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor(
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='sample_rate', full_name='voder_audio.VocodeAudioRequest.sample_rate', index=1,
number=2, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
name='vocode_params', full_name='voder_audio.VocodeAudioRequest.vocode_params', index=1,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
@@ -76,30 +162,16 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor(
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='output_rate', full_name='voder_audio.VocodeAudioRequest.output_rate', index=6,
number=7, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=7,
name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=6,
number=8, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='model_sampling_rate', full_name='voder_audio.VocodeAudioRequest.model_sampling_rate', index=8,
number=9, type=5, cpp_type=1, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
nested_types=[_VOCODEAUDIOREQUEST_VOCODEPARAMS, ],
enum_types=[
],
options=None,
@@ -109,7 +181,7 @@ _VOCODEAUDIOREQUEST = _descriptor.Descriptor(
oneofs=[
],
serialized_start=36,
serialized_end=272,
serialized_end=603,
)
@@ -139,19 +211,29 @@ _VOCODEAUDIORESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
serialized_start=274,
serialized_end=316,
serialized_start=605,
serialized_end=647,
)
_VOCODEAUDIOREQUEST_VOCODEPARAMS.containing_type = _VOCODEAUDIOREQUEST
_VOCODEAUDIOREQUEST.fields_by_name['vocode_params'].message_type = _VOCODEAUDIOREQUEST_VOCODEPARAMS
DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST
DESCRIPTOR.message_types_by_name['VocodeAudioResponse'] = _VOCODEAUDIORESPONSE
VocodeAudioRequest = _reflection.GeneratedProtocolMessageType('VocodeAudioRequest', (_message.Message,), dict(
VocodeParams = _reflection.GeneratedProtocolMessageType('VocodeParams', (_message.Message,), dict(
DESCRIPTOR = _VOCODEAUDIOREQUEST_VOCODEPARAMS,
__module__ = 'protos.audio_pb2'
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest.VocodeParams)
))
,
DESCRIPTOR = _VOCODEAUDIOREQUEST,
__module__ = 'protos.audio_pb2'
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest)
))
_sym_db.RegisterMessage(VocodeAudioRequest)
_sym_db.RegisterMessage(VocodeAudioRequest.VocodeParams)
VocodeAudioResponse = _reflection.GeneratedProtocolMessageType('VocodeAudioResponse', (_message.Message,), dict(
DESCRIPTOR = _VOCODEAUDIORESPONSE,
cycle_gan/sidecar.py
+28 -17
View File
@@ -62,7 +62,7 @@ class Converter():
frame_period = self.frame_period,
multiple = 4)
librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate)
#librosa.output.write_wav('wav_padding.wav', wav, self.sampling_rate)
f0, timeaxis, sp, ap = world_decompose(wav = wav,
fs = self.sampling_rate,
@@ -97,11 +97,11 @@ class Converter():
#librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
# TODO: Perhaps ditch this. It's probably unnecessary work.
upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
pcm_data = upsampled.astype(np.float64)
#upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
#pcm_data = upsampled.astype(np.float64)
#stereo_pcm_data = np.tile(pcm_data, (2,1)).T
#return stereo_pcm_data.astype(np.float32)
librosa.output.write_wav('model_output.wav', pcm_data, 48000)
#librosa.output.write_wav('model_output.wav', pcm_data, 48000)
#return wav
return wav_transformed
@@ -130,8 +130,8 @@ def temp_file_name(suffix='.wav'):
name = os.path.basename(temp_file.name)
return os.path.join(TEMP_DIR.name, name)
def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False,
source_rate=44100, output_rate=44100, model_sampling_rate=16000):
def convert(audio, vocode_params=None,
skip_vocode=False, save_files=False, skip_resample=False, discard_vocoded_audio=False):
#audio = np.array(audio, dtype=np.int16)
#data, samplerate = soundfile.read(audio)
#print('samplerate', samplerate)
@@ -149,6 +149,14 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis
1) Data should be float64
2) Output is BYTES!! Not floats.
3) Result is still somehow mono!?
--------------------------------------------------
demo_server.py
capture sample rate = 44100
downsample for algorithm = 16000
upsample after done = 48000
"""
audio = np.array(audio, dtype=np.float64)
@@ -161,38 +169,43 @@ def convert(audio, skip_vocode=False, save_files=False, skip_resample=False, dis
if save_files:
filename = temp_file_name('.wav')
print('----- Original wav file out: {}'.format(filename))
scipy.io.wavfile.write(filename, source_rate, audio)
scipy.io.wavfile.write(filename, vocode_params.original_source_rate, audio)
# NB: Convert the input stereo signal into mono.
# In the future the frontend should be responsible for sampling details.
#audio = audio[:, 0]
if not skip_resample:
print("Resampling audio from {} Hz to 16000 Hz".format(source_rate))
audio = librosa.resample(audio, source_rate, 16000)
if vocode_params.pre_convert_resample:
print("Resampling audio from {} Hz to {} Hz".format(vocode_params.original_source_rate,
vocode_params.pre_convert_resample_rate))
audio = librosa.resample(audio, vocode_params.original_source_rate, vocode_params.pre_convert_resample_rate)
print('resampled_audio.shape', audio.shape)
print('resampled_audio.type', audio.dtype)
if save_files:
filename = temp_file_name('.wav')
print('----- Downsampled file out: {}'.format(filename))
scipy.io.wavfile.write(filename, 16000, audio)
scipy.io.wavfile.write(filename, vocode_params.pre_convert_resample_rate, audio)
if skip_vocode:
return audio
results = converter.convert_partial(audio, conversion_direction='A2B', model_sampling_rate=model_sampling_rate)
results = converter.convert_partial(audio,
conversion_direction='A2B',
model_sampling_rate=vocode_params.model_hyperparameter_sampling_rate)
print('results.type', type(results))
print('results.len', len(results))
"""
if output_rate != 16000:
print("Resampling output audio from {} Hz to 16000 Hz".format(source_rate))
print("Resampling output audio from {} Hz to 16000 Hz".format(vocode_params.source_rate))
#consume_rate = 68000 # Experimentally determined for Rust lib 'CPAL'
results = librosa.resample(results, 16000, output_rate)
if save_files:
filename = temp_file_name('.wav')
print('----- Upsampled (transformed) file out: {}'.format(filename))
scipy.io.wavfile.write(filename, output_rate, results)
"""
if discard_vocoded_audio:
return audio
@@ -215,17 +228,15 @@ def main():
message = socket.recv()
vocode_request = VocodeAudioRequest.FromString(message)
vocode_params = vocode_request.vocode_params
queue.extend(vocode_request.float_audio)
if len(queue) >= vocode_request.buffer_size_minimum:
#results = queue[:]
results = convert(queue,
source_rate=vocode_request.sample_rate,
output_rate=vocode_request.output_rate,
model_sampling_rate=vocode_request.model_sampling_rate,
vocode_params=vocode_params,
skip_vocode=vocode_request.skip_vocode,
skip_resample=vocode_request.skip_resample,
save_files=vocode_request.save_files,
discard_vocoded_audio=vocode_request.discard_vocoded_audio)
queue = []
protos/audio.proto
+27 -3
View File
@@ -4,9 +4,33 @@ package voder_audio;
message VocodeAudioRequest {
repeated float float_audio = 1;
message VocodeParams {
// The initial sample rate coming from the microphone.
int32 initial_sample_rate = 1;
// Original sample rate
int32 original_source_rate = 9;
// Resample before passing to the algorithm?
bool pre_convert_resample = 2;
int32 pre_convert_resample_rate = 3;
bool pre_convert_resample_save_file = 4;
// The hard-coded hyperparameter of the model.
// Probably don't want to change from 16000.
int32 model_hyperparameter_sampling_rate = 5;
// Resample after passing to the algorithm?
bool post_convert_resample = 6;
int32 post_convert_resample_rate = 7;
bool post_convert_resample_save_file = 8;
}
VocodeParams vocode_params = 10;
// We'll resend these params for every batch, but will only use
// the last batch's settings. Kind of a dumb API, but it'll work.
int32 sample_rate = 2;
//int32 sample_rate = 2;
bool skip_resample = 3;
bool skip_vocode = 4;
@@ -18,13 +42,13 @@ message VocodeAudioRequest {
int32 buffer_size_minimum = 6;
// Requested output rate
int32 output_rate = 7;
//int32 output_rate = 7;
// Discard the vocoded audio and return the original
bool discard_vocoded_audio = 8;
// Rate param for the model
int32 model_sampling_rate = 9;
//int32 model_sampling_rate = 9;
}
message VocodeAudioResponse {
src/main.rs
+20 -4
View File
@@ -41,6 +41,7 @@ use std::process::exit;
use std::sync::Arc;
use std::thread;
use zmq::{Error, Socket};
use protos::voder_audio::vocode_audio_request::VocodeParams;
fn main() {
//print_version();
@@ -101,16 +102,31 @@ fn run_cpal_audio() -> Result<(), failure::Error> {
};
let mut vocode_request = VocodeAudioRequest::default();
vocode_request.sample_rate = 16000;
vocode_request.output_rate = 16000;
vocode_request.model_sampling_rate = 88000;
//vocode_request.sample_rate = 16000;
//vocode_request.output_rate = 16000;
//vocode_request.model_sampling_rate = 88000; // This is close!
//vocode_request.model_sampling_rate = 88000;
vocode_request.skip_resample = false;
vocode_request.skip_vocode = false;
vocode_request.discard_vocoded_audio = false;
vocode_request.save_files = false;
vocode_request.buffer_size_minimum = 100000;
//vocode_request.buffer_size_minimum = 5000; // AWFUL. SO CHOPPY.
vocode_request.buffer_size_minimum = 50000; // Practically real time, but lots more phase distortion.
//vocode_request.buffer_size_minimum = 70000; // Sounds pretty good.
//vocode_request.buffer_size_minimum = 100000; // This sounds good! A bit slow ~3seconds.
//vocode_request.buffer_size_minimum = 200000;
vocode_request.float_audio = drained.clone();
let mut vocode_params = VocodeParams::default();
vocode_params.original_source_rate = 16000;
vocode_params.pre_convert_resample = true;
vocode_params.pre_convert_resample_rate = 16000;
vocode_params.model_hyperparameter_sampling_rate = 88000;
vocode_params.post_convert_resample = false;
vocode_params.post_convert_resample_rate = 16000;
vocode_request.vocode_params = Some(vocode_params);
let mut encoded_bytes = Vec::with_capacity(vocode_request.encoded_len());
vocode_request.encode(&mut encoded_bytes).unwrap();
src/protos/voder_audio.rs
+38 -10
View File
@@ -2,10 +2,12 @@
pub struct VocodeAudioRequest {
#[prost(float, repeated, tag="1")]
pub float_audio: ::std::vec::Vec<f32>,
/// We'll resend these params for every batch, but will only use
/// the last batch's settings. Kind of a dumb API, but it'll work.
#[prost(int32, tag="2")]
pub sample_rate: i32,
#[prost(message, optional, tag="10")]
pub vocode_params: ::std::option::Option<vocode_audio_request::VocodeParams>,
// We'll resend these params for every batch, but will only use
// the last batch's settings. Kind of a dumb API, but it'll work.
//int32 sample_rate = 2;
#[prost(bool, tag="3")]
pub skip_resample: bool,
#[prost(bool, tag="4")]
@@ -16,15 +18,41 @@ pub struct VocodeAudioRequest {
/// How big we let the buffer grow before running 'convert'.
#[prost(int32, tag="6")]
pub buffer_size_minimum: i32,
/// Requested output rate
#[prost(int32, tag="7")]
pub output_rate: i32,
// Requested output rate
//int32 output_rate = 7;
/// Discard the vocoded audio and return the original
#[prost(bool, tag="8")]
pub discard_vocoded_audio: bool,
/// Rate param for the model
#[prost(int32, tag="9")]
pub model_sampling_rate: i32,
}
pub mod vocode_audio_request {
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct VocodeParams {
/// The initial sample rate coming from the microphone.
#[prost(int32, tag="1")]
pub initial_sample_rate: i32,
/// Original sample rate
#[prost(int32, tag="9")]
pub original_source_rate: i32,
/// Resample before passing to the algorithm?
#[prost(bool, tag="2")]
pub pre_convert_resample: bool,
#[prost(int32, tag="3")]
pub pre_convert_resample_rate: i32,
#[prost(bool, tag="4")]
pub pre_convert_resample_save_file: bool,
/// The hard-coded hyperparameter of the model.
/// Probably don't want to change from 16000.
#[prost(int32, tag="5")]
pub model_hyperparameter_sampling_rate: i32,
/// Resample after passing to the algorithm?
#[prost(bool, tag="6")]
pub post_convert_resample: bool,
#[prost(int32, tag="7")]
pub post_convert_resample_rate: i32,
#[prost(bool, tag="8")]
pub post_convert_resample_save_file: bool,
}
}
#[derive(Clone, PartialEq, ::prost::Message)]
pub struct VocodeAudioResponse {