From e71a3772021d47bbf6a9c4f1e38420ba316f7e8c Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 30 May 2020 14:05:19 +0200 Subject: [PATCH 01/68] add a non-existing value to avg values --- utils/generic_utils.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 1c7dd5e4..5a811907 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -197,14 +197,19 @@ class KeepAverage(): self.iters[name] = init_iter def update_value(self, name, value, weighted_avg=False): - if weighted_avg: - self.avg_values[name] = 0.99 * self.avg_values[name] + 0.01 * value - self.iters[name] += 1 + if name not in self.avg_values: + # add value if not exist before + self.add_value(name, init_val=value) else: - self.avg_values[name] = self.avg_values[name] * \ - self.iters[name] + value - self.iters[name] += 1 - self.avg_values[name] /= self.iters[name] + # else update existing value + if weighted_avg: + self.avg_values[name] = 0.99 * self.avg_values[name] + 0.01 * value + self.iters[name] += 1 + else: + self.avg_values[name] = self.avg_values[name] * \ + self.iters[name] + value + self.iters[name] += 1 + self.avg_values[name] /= self.iters[name] def add_values(self, name_dict): for key, value in name_dict.items(): From 5116a20c86608a2790b1b7cecf0765a003134218 Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 30 May 2020 18:07:01 +0200 Subject: [PATCH 02/68] differentiate modules on TB with a prefix --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index e4963ee7..82f91cd5 100644 --- a/train.py +++ b/train.py @@ -667,7 +667,7 @@ if __name__ == '__main__': os.chmod(OUT_PATH, 0o775) LOG_DIR = OUT_PATH - tb_logger = TensorboardLogger(LOG_DIR) + tb_logger = TensorboardLogger(LOG_DIR, model_name='TTS') # write model desc to tensorboard tb_logger.tb_add_text('model-description', c['run_description'], 0) From 0372c5ce5d066667102193133aa46133037c1d09 Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 30 May 2020 18:08:26 +0200 Subject: [PATCH 03/68] set state dict using direct state_dict dict --- utils/generic_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 5a811907..5b135061 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -106,15 +106,15 @@ def sequence_mask(sequence_length, max_len=None): return seq_range_expand < seq_length_expand -def set_init_dict(model_dict, checkpoint, c): +def set_init_dict(model_dict, checkpoint_state, c): # Partial initialization: if there is a mismatch with new and old layer, it is skipped. - for k, v in checkpoint['model'].items(): + for k, v in checkpoint_state.items(): if k not in model_dict: print(" | > Layer missing in the model definition: {}".format(k)) # 1. filter out unnecessary keys pretrained_dict = { k: v - for k, v in checkpoint['model'].items() if k in model_dict + for k, v in checkpoint_state.items() if k in model_dict } # 2. filter out different size layers pretrained_dict = { From 4ecd47e072ab594dc0c7af952b341a8f6f8090bb Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 30 May 2020 18:08:56 +0200 Subject: [PATCH 04/68] use model_name prefix in tb_logger --- utils/tensorboard_logger.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/utils/tensorboard_logger.py b/utils/tensorboard_logger.py index 15fe04e4..cbf68ad6 100644 --- a/utils/tensorboard_logger.py +++ b/utils/tensorboard_logger.py @@ -3,7 +3,8 @@ from tensorboardX import SummaryWriter class TensorboardLogger(object): - def __init__(self, log_dir): + def __init__(self, log_dir, model_name): + self.model_name = model_name self.writer = SummaryWriter(log_dir) self.train_stats = {} self.eval_stats = {} @@ -50,31 +51,31 @@ class TensorboardLogger(object): traceback.print_exc() def tb_train_iter_stats(self, step, stats): - self.dict_to_tb_scalar("TrainIterStats", stats, step) + self.dict_to_tb_scalar(f"{self.model_name}_TrainIterStats", stats, step) def tb_train_epoch_stats(self, step, stats): - self.dict_to_tb_scalar("TrainEpochStats", stats, step) + self.dict_to_tb_scalar(f"{self.model_name}_TrainEpochStats", stats, step) def tb_train_figures(self, step, figures): - self.dict_to_tb_figure("TrainFigures", figures, step) + self.dict_to_tb_figure(f"{self.model_name}_TrainFigures", figures, step) def tb_train_audios(self, step, audios, sample_rate): - self.dict_to_tb_audios("TrainAudios", audios, step, sample_rate) + self.dict_to_tb_audios(f"{self.model_name}_TrainAudios", audios, step, sample_rate) def tb_eval_stats(self, step, stats): - self.dict_to_tb_scalar("EvalStats", stats, step) + self.dict_to_tb_scalar(f"{self.model_name}_EvalStats", stats, step) def tb_eval_figures(self, step, figures): - self.dict_to_tb_figure("EvalFigures", figures, step) + self.dict_to_tb_figure(f"{self.model_name}_EvalFigures", figures, step) def tb_eval_audios(self, step, audios, sample_rate): - self.dict_to_tb_audios("EvalAudios", audios, step, sample_rate) + self.dict_to_tb_audios(f"{self.model_name}_EvalAudios", audios, step, sample_rate) def tb_test_audios(self, step, audios, sample_rate): - self.dict_to_tb_audios("TestAudios", audios, step, sample_rate) + self.dict_to_tb_audios(f"{self.model_name}_TestAudios", audios, step, sample_rate) def tb_test_figures(self, step, figures): - self.dict_to_tb_figure("TestFigures", figures, step) + self.dict_to_tb_figure(f"{self.model_name}_TestFigures", figures, step) def tb_add_text(self, title, text, step): self.writer.add_text(title, text, step) From d47c9987dbaa2dbd129b3f897c6fb45b392ae455 Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 30 May 2020 18:09:25 +0200 Subject: [PATCH 05/68] initial commit intro. to vocoder submodule --- vocoder/README.md | 35 + vocoder/__init__.py | 0 vocoder/compute_tts_features.py | 0 vocoder/configs/melgan_config.json | 138 ++++ vocoder/datasets/__init__.py | 0 vocoder/datasets/gan_dataset.py | 133 ++++ vocoder/datasets/preprocess.py | 16 + vocoder/layers/__init__.py | 0 vocoder/layers/losses.py | 273 +++++++ vocoder/layers/melgan.py | 48 ++ vocoder/layers/pqmf.py | 55 ++ vocoder/layers/pqmf2.py | 127 ++++ vocoder/layers/qmf.dat | 640 +++++++++++++++++ vocoder/models/__init__.py | 0 vocoder/models/melgan_discriminator.py | 80 +++ vocoder/models/melgan_generator.py | 91 +++ .../models/melgan_multiscale_discriminator.py | 41 ++ vocoder/models/multiband_melgan_generator.py | 38 + vocoder/models/random_window_discriminator.py | 225 ++++++ vocoder/notebooks/Untitled.ipynb | 678 ++++++++++++++++++ vocoder/notebooks/Untitled1.ipynb | 6 + vocoder/pqmf_output.wav | Bin 0 -> 83812 bytes vocoder/tests/__init__.py | 1 + vocoder/tests/test_config.json | 24 + vocoder/tests/test_datasets.py | 95 +++ vocoder/tests/test_losses.py | 62 ++ vocoder/tests/test_melgan_discriminator.py | 26 + vocoder/tests/test_melgan_generator.py | 15 + vocoder/tests/test_pqmf.py | 33 + vocoder/tests/test_rwd.py | 21 + vocoder/train.py | 585 +++++++++++++++ vocoder/utils/__init__.py | 0 vocoder/utils/console_logger.py | 97 +++ vocoder/utils/generic_utils.py | 102 +++ vocoder/utils/io.py | 52 ++ 35 files changed, 3737 insertions(+) create mode 100644 vocoder/README.md create mode 100644 vocoder/__init__.py create mode 100644 vocoder/compute_tts_features.py create mode 100644 vocoder/configs/melgan_config.json create mode 100644 vocoder/datasets/__init__.py create mode 100644 vocoder/datasets/gan_dataset.py create mode 100644 vocoder/datasets/preprocess.py create mode 100644 vocoder/layers/__init__.py create mode 100644 vocoder/layers/losses.py create mode 100644 vocoder/layers/melgan.py create mode 100644 vocoder/layers/pqmf.py create mode 100644 vocoder/layers/pqmf2.py create mode 100644 vocoder/layers/qmf.dat create mode 100644 vocoder/models/__init__.py create mode 100644 vocoder/models/melgan_discriminator.py create mode 100644 vocoder/models/melgan_generator.py create mode 100644 vocoder/models/melgan_multiscale_discriminator.py create mode 100644 vocoder/models/multiband_melgan_generator.py create mode 100644 vocoder/models/random_window_discriminator.py create mode 100644 vocoder/notebooks/Untitled.ipynb create mode 100644 vocoder/notebooks/Untitled1.ipynb create mode 100644 vocoder/pqmf_output.wav create mode 100644 vocoder/tests/__init__.py create mode 100644 vocoder/tests/test_config.json create mode 100644 vocoder/tests/test_datasets.py create mode 100644 vocoder/tests/test_losses.py create mode 100644 vocoder/tests/test_melgan_discriminator.py create mode 100644 vocoder/tests/test_melgan_generator.py create mode 100644 vocoder/tests/test_pqmf.py create mode 100644 vocoder/tests/test_rwd.py create mode 100644 vocoder/train.py create mode 100644 vocoder/utils/__init__.py create mode 100644 vocoder/utils/console_logger.py create mode 100644 vocoder/utils/generic_utils.py create mode 100644 vocoder/utils/io.py diff --git a/vocoder/README.md b/vocoder/README.md new file mode 100644 index 00000000..48fc24ee --- /dev/null +++ b/vocoder/README.md @@ -0,0 +1,35 @@ +# Mozilla TTS Vocoders (Experimental) + +We provide here different vocoder implementations which can be combined with our TTS models to enable "FASTER THAN REAL-TIME" end-to-end TTS stack. + +Currently, there are implementations of the following models. + +- Melgan +- MultiBand-Melgan +- GAN-TTS (Discriminator Only) + +It is also very easy to adapt different vocoder models as we provide here a flexible and modular (but not too modular) framework. + +## Training a model + +You can see here an example (Soon)[Colab Notebook]() training MelGAN with LJSpeech dataset. + +In order to train a new model, you need to collecto all your wav files under a common parent folder and give this path to `data_path` field in '''config.json''' + +You need to define other relevant parameters in your ```config.json``` and then start traning with the following command from Mozilla TTS root path. + +```CUDA_VISIBLE_DEVICES='1' python vocoder/train.py --config_path path/to/config.json``` + +Exampled config files can be found under `vocoder/configs/` folder. + +You can continue a previous training by the following command. + +```CUDA_VISIBLE_DEVICES='1' python vocoder/train.py --continue_path path/to/your/model/folder``` + +You can fine-tune a pre-trained model by the following command. + +```CUDA_VISIBLE_DEVICES='1' python vocoder/train.py --restore_path path/to/your/model.pth.tar``` + +Restoring a model starts a new training in a different output folder. It only restores model weights with the given checkpoint file. However, continuing a training starts from the same conditions the previous training run left off. + +You can also follow your training runs on Tensorboard as you do with our TTS models. \ No newline at end of file diff --git a/vocoder/__init__.py b/vocoder/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/compute_tts_features.py b/vocoder/compute_tts_features.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json new file mode 100644 index 00000000..9a3ded37 --- /dev/null +++ b/vocoder/configs/melgan_config.json @@ -0,0 +1,138 @@ +{ + "run_name": "melgan", + "run_description": "melgan initial run", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // DISTRIBUTED TRAINING + // "distributed":{ + // "backend": "nccl", + // "url": "tcp:\/\/localhost:54321" + // }, + + // MODEL PARAMETERS + "use_pqmf": true, + + // LOSS PARAMETERS + "use_stft_loss": true, + "use_mse_gan_loss": true, + "use_hinge_gan_loss": false, + "use_feat_match_loss": false, // use only with melgan discriminators + + "stft_loss_alpha": 1, + "mse_gan_loss_alpha": 1, + "hinge_gan_loss_alpha": 1, + "feat_match_loss_alpha": 10.0, + + "stft_loss_params": { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }, + "target_loss": "avg_G_loss", // loss value to pick the best model + + // DISCRIMINATOR + "discriminator_model": "melgan_multiscale_discriminator", + "discriminator_model_params":{ + "base_channels": 16, + "max_channels":1024, + "downsample_factors":[4, 4, 4, 4] + }, + "steps_to_start_discriminator": 100000, // steps required to start GAN trainining.1 + + // "discriminator_model": "random_window_discriminator", + // "discriminator_model_params":{ + // "uncond_disc_donwsample_factors": [8, 4], + // "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]], + // "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], + // "window_sizes": [512, 1024, 2048, 4096, 8192] + // }, + + + // GENERATOR + "generator_model": "multiband_melgan_generator", + "generator_model_params": { + "upsample_factors":[2 ,2, 4, 4], + "num_res_blocks": 4 + }, + + // DATASET + "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", + "seq_len": 16384, + "pad_short": 2000, + "conv_pad": 0, + "use_noise_augment": true, + "use_cache": true, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": true, // use noam warmup and lr schedule. + "grad_clip": 1.0, // upper limit for gradients for clipping. + "epochs": 1000, // total number of epochs to train. + "wd": 0.000001, // Weight decay weight. + "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_disc": 0.0001, + "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "warmup_steps_disc": 4000, + "gen_clip_grad": 10.0, + "disc_clip_grad": 10.0, + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log traning on console. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "eval_split_size": 10, + + // PATHS + "output_path": "/home/erogol/Models/LJSpeech/" +} + diff --git a/vocoder/datasets/__init__.py b/vocoder/datasets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/datasets/gan_dataset.py b/vocoder/datasets/gan_dataset.py new file mode 100644 index 00000000..10d36cab --- /dev/null +++ b/vocoder/datasets/gan_dataset.py @@ -0,0 +1,133 @@ +import os +import glob +import torch +import random +import numpy as np +from torch.utils.data import Dataset, DataLoader +from multiprocessing import Manager + + +def create_dataloader(hp, args, train): + dataset = MelFromDisk(hp, args, train) + + if train: + return DataLoader(dataset=dataset, + batch_size=hp.train.batch_size, + shuffle=True, + num_workers=hp.train.num_workers, + pin_memory=True, + drop_last=True) + else: + return DataLoader(dataset=dataset, + batch_size=1, + shuffle=False, + num_workers=hp.train.num_workers, + pin_memory=True, + drop_last=False) + + +class GANDataset(Dataset): + """ + GAN Dataset searchs for all the wav files under root path + and converts them to acoustic features on the fly and returns + random segments of (audio, feature) couples. + """ + def __init__(self, + ap, + items, + seq_len, + hop_len, + pad_short, + conv_pad=2, + is_training=True, + return_segments=True, + use_noise_augment=False, + use_cache=False, + verbose=False): + + self.ap = ap + self.item_list = items + self.seq_len = seq_len + self.hop_len = hop_len + self.pad_short = pad_short + self.conv_pad = conv_pad + self.is_training = is_training + self.return_segments = return_segments + self.use_cache = use_cache + self.use_noise_augment = use_noise_augment + + assert seq_len % hop_len == 0, " [!] seq_len has to be a multiple of hop_len." + self.feat_frame_len = seq_len // hop_len + (2 * conv_pad) + + # map G and D instances + self.G_to_D_mappings = [i for i in range(len(self.item_list))] + self.shuffle_mapping() + + # cache acoustic features + if use_cache: + self.create_feature_cache() + + + + def create_feature_cache(self): + self.manager = Manager() + self.cache = self.manager.list() + self.cache += [None for _ in range(len(self.item_list))] + + def find_wav_files(self, path): + return glob.glob(os.path.join(path, '**', '*.wav'), recursive=True) + + def __len__(self): + return len(self.item_list) + + def __getitem__(self, idx): + """ Return different items for Generator and Discriminator and + cache acoustic features """ + if self.return_segments: + idx2 = self.G_to_D_mappings[idx] + item1 = self.load_item(idx) + item2 = self.load_item(idx2) + return item1, item2 + else: + item1 = self.load_item(idx) + return item1 + + def shuffle_mapping(self): + random.shuffle(self.G_to_D_mappings) + + def load_item(self, idx): + """ load (audio, feat) couple """ + wavpath = self.item_list[idx] + # print(wavpath) + + if self.use_cache and self.cache[idx] is not None: + audio, mel = self.cache[idx] + else: + audio = self.ap.load_wav(wavpath) + mel = self.ap.melspectrogram(audio) + + if len(audio) < self.seq_len + self.pad_short: + audio = np.pad(audio, (0, self.seq_len + self.pad_short - len(audio)), \ + mode='constant', constant_values=0.0) + + # correct the audio length wrt padding applied in stft + audio = np.pad(audio, (0, self.hop_len), mode="edge") + audio = audio[:mel.shape[-1] * self.hop_len] + assert mel.shape[-1] * self.hop_len == audio.shape[-1], f' [!] {mel.shape[-1] * self.hop_len} vs {audio.shape[-1]}' + + audio = torch.from_numpy(audio).float().unsqueeze(0) + mel = torch.from_numpy(mel).float().squeeze(0) + + if self.return_segments: + max_mel_start = mel.shape[1] - self.feat_frame_len + mel_start = random.randint(0, max_mel_start) + mel_end = mel_start + self.feat_frame_len + mel = mel[:, mel_start:mel_end] + + audio_start = mel_start * self.hop_len + audio = audio[:, audio_start:audio_start + + self.seq_len] + + if self.use_noise_augment and self.is_training and self.return_segments: + audio = audio + (1 / 32768) * torch.randn_like(audio) + return (mel, audio) diff --git a/vocoder/datasets/preprocess.py b/vocoder/datasets/preprocess.py new file mode 100644 index 00000000..01e01e3e --- /dev/null +++ b/vocoder/datasets/preprocess.py @@ -0,0 +1,16 @@ +import glob +import os + +import numpy as np + + +def find_wav_files(data_path): + wav_paths = glob.glob(os.path.join(data_path, '**', '*.wav'), recursive=True) + return wav_paths + + +def load_wav_data(data_path, eval_split_size): + wav_paths = find_wav_files(data_path) + np.random.seed(0) + np.random.shuffle(wav_paths) + return wav_paths[:eval_split_size], wav_paths[eval_split_size:] diff --git a/vocoder/layers/__init__.py b/vocoder/layers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py new file mode 100644 index 00000000..11985629 --- /dev/null +++ b/vocoder/layers/losses.py @@ -0,0 +1,273 @@ +import torch + +from torch import nn +from torch.nn import functional as F + + +class TorchSTFT(): + def __init__(self, n_fft, hop_length, win_length, window='hann_window'): + self.n_fft = n_fft + self.hop_length = hop_length + self.win_length = win_length + self.window = getattr(torch, window)(win_length) + + def __call__(self, x): + # B x D x T x 2 + o = torch.stft(x, + self.n_fft, + self.hop_length, + self.win_length, + self.window, + center=True, + pad_mode="constant", # compatible with audio.py + normalized=False, + onesided=True) + M = o[:, :, :, 0] + P = o[:, :, :, 1] + return torch.sqrt(torch.clamp(M ** 2 + P ** 2, min=1e-8)) + + +################################# +# GENERATOR LOSSES +################################# + + +class STFTLoss(nn.Module): + def __init__(self, n_fft, hop_length, win_length): + super(STFTLoss, self).__init__() + self.n_fft = n_fft + self.hop_length = hop_length + self.win_length = win_length + self.stft = TorchSTFT(n_fft, hop_length, win_length) + + def forward(self, y_hat, y): + y_hat_M = self.stft(y_hat) + y_M = self.stft(y) + # magnitude loss + loss_mag = F.l1_loss(torch.log(y_M), torch.log(y_hat_M)) + # spectral convergence loss + loss_sc = torch.norm(y_M - y_hat_M, p="fro") / torch.norm(y_M, p="fro") + return loss_mag, loss_sc + + +class MultiScaleSTFTLoss(torch.nn.Module): + def __init__(self, + n_ffts=[1024, 2048, 512], + hop_lengths=[120, 240, 50], + win_lengths=[600, 1200, 240]): + super(MultiScaleSTFTLoss, self).__init__() + self.loss_funcs = torch.nn.ModuleList() + for idx in range(len(n_ffts)): + self.loss_funcs.append(STFTLoss(n_ffts[idx], hop_lengths[idx], win_lengths[idx])) + + def forward(self, y_hat, y): + N = len(self.loss_funcs) + loss_sc = 0 + loss_mag = 0 + for f in self.loss_funcs: + lm, lsc = f(y_hat, y) + loss_mag += lm + loss_sc += lsc + loss_sc /= N + loss_mag /= N + return loss_mag, loss_sc + + +class MSEGLoss(nn.Module): + """ Mean Squared Generator Loss """ + def __init__(self,): + super(MSEGLoss, self).__init__() + + def forward(self, score_fake, ): + loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) + return loss_fake + + +class HingeGLoss(nn.Module): + """ Hinge Discriminator Loss """ + def __init__(self,): + super(HingeGLoss, self).__init__() + + def forward(self, score_fake, score_real): + loss_fake = torch.mean(F.relu(1. + score_fake)) + return loss_fake + + +################################## +# DISCRIMINATOR LOSSES +################################## + + +class MSEDLoss(nn.Module): + """ Mean Squared Discriminator Loss """ + def __init__(self,): + super(MSEDLoss, self).__init__() + + def forward(self, score_fake, score_real): + loss_real = torch.mean(torch.sum(torch.pow(score_real - 1.0, 2), dim=[1, 2])) + loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) + loss_d = loss_real + loss_fake + return loss_d, loss_real, loss_fake + + +class HingeDLoss(nn.Module): + """ Hinge Discriminator Loss """ + def __init__(self,): + super(HingeDLoss, self).__init__() + + def forward(self, score_fake, score_real): + loss_real = torch.mean(F.relu(1. - score_real)) + loss_fake = torch.mean(F.relu(1. + score_fake)) + loss_d = loss_real + loss_fake + return loss_d, loss_real, loss_fake + + +class MelganFeatureLoss(nn.Module): + def __init__(self, ): + super(MelganFeatureLoss, self).__init__() + + def forward(self, fake_feats, real_feats): + loss_feats = 0 + for fake_feat, real_feat in zip(fake_feats, real_feats): + loss_feats += hp.model.feat_match * torch.mean(torch.abs(fake_feat - real_feat)) + return loss_feats + + +################################## +# LOSS WRAPPERS +################################## + + +class GeneratorLoss(nn.Module): + def __init__(self, C): + super(GeneratorLoss, self).__init__() + assert C.use_mse_gan_loss and C.use_hinge_gan_loss == False,\ + " [!] Cannot use HingeGANLoss and MSEGANLoss together." + + self.use_stft_loss = C.use_stft_loss + self.use_mse_gan_loss = C.use_mse_gan_loss + self.use_hinge_gan_loss = C.use_hinge_gan_loss + self.use_feat_match_loss = C.use_feat_match_loss + + self.stft_loss_alpha = C.stft_loss_alpha + self.mse_gan_loss_alpha = C.mse_gan_loss_alpha + self.hinge_gan_loss_alpha = C.hinge_gan_loss_alpha + self.feat_match_loss_alpha = C.feat_match_loss_alpha + + if C.use_stft_loss: + self.stft_loss = MultiScaleSTFTLoss(**C.stft_loss_params) + if C.use_mse_gan_loss: + self.mse_loss = MSEGLoss() + if C.use_hinge_gan_loss: + self.hinge_loss = HingeGLoss() + if C.use_feat_match_loss: + self.feat_match_loss = MelganFeatureLoss() + + def forward(self, y_hat=None, y=None, scores_fake=None, feats_fake=None, feats_real=None): + loss = 0 + return_dict = {} + + # STFT Loss + if self.use_stft_loss: + stft_loss_mg, stft_loss_sc = self.stft_loss(y_hat.squeeze(1), y.squeeze(1)) + return_dict['G_stft_loss_mg'] = stft_loss_mg + return_dict['G_stft_loss_sc'] = stft_loss_sc + loss += self.stft_loss_alpha * (stft_loss_mg + stft_loss_sc) + + # Fake Losses + if self.use_mse_gan_loss and scores_fake is not None: + mse_fake_loss = 0 + if isinstance(scores_fake, list): + for score_fake in scores_fake: + fake_loss = self.mse_loss(score_fake) + mse_fake_loss += fake_loss + else: + fake_loss = self.mse_loss(scores_fake) + mse_fake_loss = fake_loss + return_dict['G_mse_fake_loss'] = mse_fake_loss + loss += self.mse_gan_loss_alpha * mse_fake_loss + + if self.use_hinge_gan_loss and not scores_fake is not None: + hinge_fake_loss = 0 + if isinstance(scores_fake, list): + for score_fake in scores_fake: + fake_loss = self.hinge_loss(score_fake) + hinge_fake_loss += fake_loss + else: + fake_loss = self.hinge_loss(scores_fake) + hinge_fake_loss = fake_loss + return_dict['G_hinge_fake_loss'] = hinge_fake_loss + loss += self.hinge_gan_loss_alpha * hinge_fake_loss + + # Feature Matching Loss + if self.use_feat_match_loss and not feats_fake: + feat_match_loss = self.feat_match_loss(feats_fake, feats_real) + return_dict['G_feat_match_loss'] = feat_match_loss + loss += self.feat_match_loss_alpha * feat_match_loss + return_dict['G_loss'] = loss + return return_dict + + +class DiscriminatorLoss(nn.Module): + def __init__(self, C): + super(DiscriminatorLoss, self).__init__() + assert C.use_mse_gan_loss and C.use_hinge_gan_loss == False,\ + " [!] Cannot use HingeGANLoss and MSEGANLoss together." + + self.use_mse_gan_loss = C.use_mse_gan_loss + self.use_hinge_gan_loss = C.use_hinge_gan_loss + + self.mse_gan_loss_alpha = C.mse_gan_loss_alpha + self.hinge_gan_loss_alpha = C.hinge_gan_loss_alpha + + if C.use_mse_gan_loss: + self.mse_loss = MSEDLoss() + if C.use_hinge_gan_loss: + self.hinge_loss = HingeDLoss() + + def forward(self, scores_fake, scores_real): + loss = 0 + return_dict = {} + + if self.use_mse_gan_loss: + mse_gan_loss = 0 + mse_gan_real_loss = 0 + mse_gan_fake_loss = 0 + if isinstance(scores_fake, list): + for score_fake, score_real in zip(scores_fake, scores_real): + total_loss, real_loss, fake_loss = self.mse_loss(score_fake, score_real) + mse_gan_loss += total_loss + mse_gan_real_loss += real_loss + mse_gan_fake_loss += fake_loss + else: + total_loss, real_loss, fake_loss = self.mse_loss(scores_fake, scores_real) + mse_gan_loss = total_loss + mse_gan_real_loss = real_loss + mse_gan_fake_loss = fake_loss + return_dict['D_mse_gan_loss'] = mse_gan_loss + return_dict['D_mse_gan_real_loss'] = mse_gan_real_loss + return_dict['D_mse_gan_fake_loss'] = mse_gan_fake_loss + loss += self.mse_gan_loss_alpha * mse_gan_loss + + if self.use_hinge_gan_loss: + hinge_gan_loss = 0 + hinge_gan_real_loss = 0 + hinge_gan_fake_loss = 0 + if isinstance(scores_fake, list): + for score_fake, score_real in zip(scores_fake, scores_real): + total_loss, real_loss, fake_loss = self.hinge_loss(score_fake, score_real) + hinge_gan_loss += total_loss + hinge_gan_real_loss += real_loss + hinge_gan_fake_loss += fake_loss + else: + total_loss, real_loss, fake_loss = self.hinge_loss(scores_fake, scores_real) + hinge_gan_loss = total_loss + hinge_gan_real_loss = real_loss + hinge_gan_fake_loss = fake_loss + return_dict['D_hinge_gan_loss'] = hinge_gan_loss + return_dict['D_hinge_gan_real_loss'] = hinge_gan_real_loss + return_dict['D_hinge_gan_fake_loss'] = hinge_gan_fake_loss + loss += self.hinge_gan_loss_alpha * hinge_gan_loss + + return_dict['D_loss'] = loss + return return_dict \ No newline at end of file diff --git a/vocoder/layers/melgan.py b/vocoder/layers/melgan.py new file mode 100644 index 00000000..cda0413c --- /dev/null +++ b/vocoder/layers/melgan.py @@ -0,0 +1,48 @@ +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.utils import weight_norm + + +class ResidualStack(nn.Module): + def __init__(self, channels, num_res_blocks, kernel_size): + super(ResidualStack, self).__init__() + + assert (kernel_size - 1) % 2 == 0, " [!] kernel_size has to be odd." + base_padding = (kernel_size - 1) // 2 + + self.blocks = nn.ModuleList() + for idx in range(num_res_blocks): + layer_kernel_size = kernel_size + layer_dilation = layer_kernel_size**idx + layer_padding = base_padding * layer_dilation + self.blocks += [nn.Sequential( + nn.LeakyReLU(0.2), + nn.ReflectionPad1d(layer_padding), + weight_norm( + nn.Conv1d(channels, + channels, + kernel_size=kernel_size, + dilation=layer_padding, + bias=True)), + nn.LeakyReLU(0.2), + weight_norm( + nn.Conv1d(channels, channels, kernel_size=1, bias=True)), + )] + + self.shortcuts = nn.ModuleList([ + weight_norm(nn.Conv1d(channels, channels, kernel_size=1, + bias=True)) for i in range(num_res_blocks) + ]) + + def forward(self, x): + for block, shortcut in zip(self.blocks, self.shortcuts): + x = shortcut(x) + block(x) + return x + + def remove_weight_norm(self): + for block, shortcut in zip(self.blocks, self.shortcuts): + nn.utils.remove_weight_norm(block[2]) + nn.utils.remove_weight_norm(block[4]) + nn.utils.remove_weight_norm(shortcut) diff --git a/vocoder/layers/pqmf.py b/vocoder/layers/pqmf.py new file mode 100644 index 00000000..f438ea00 --- /dev/null +++ b/vocoder/layers/pqmf.py @@ -0,0 +1,55 @@ +"""Pseudo QMF modules.""" + +import numpy as np +import torch +import torch.nn.functional as F + +from scipy import signal as sig + + +# adapted from +# https://github.com/kan-bayashi/ParallelWaveGAN/tree/master/parallel_wavegan +class PQMF(torch.nn.Module): + def __init__(self, N=4, taps=62, cutoff=0.15, beta=9.0): + super(PQMF, self).__init__() + + self.N = N + self.taps = taps + self.cutoff = cutoff + self.beta = beta + + QMF = sig.firwin(taps + 1, cutoff, window=('kaiser', beta)) + H = np.zeros((N, len(QMF))) + G = np.zeros((N, len(QMF))) + for k in range(N): + constant_factor = (2 * k + 1) * (np.pi / + (2 * N)) * (np.arange(taps + 1) - + ((taps - 1) / 2)) + phase = (-1)**k * np.pi / 4 + H[k] = 2 * QMF * np.cos(constant_factor + phase) + + G[k] = 2 * QMF * np.cos(constant_factor - phase) + + H = torch.from_numpy(H[:, None, :]).float() + G = torch.from_numpy(G[None, :, :]).float() + + self.register_buffer("H", H) + self.register_buffer("G", G) + + updown_filter = torch.zeros((N, N, N)).float() + for k in range(N): + updown_filter[k, k, 0] = 1.0 + self.register_buffer("updown_filter", updown_filter) + self.N = N + + self.pad_fn = torch.nn.ConstantPad1d(taps // 2, 0.0) + + def analysis(self, x): + return F.conv1d(x, self.H, padding=self.taps // 2, stride=self.N) + + def synthesis(self, x): + x = F.conv_transpose1d(x, + self.updown_filter * self.N, + stride=self.N) + x = F.conv1d(x, self.G, padding=self.taps // 2) + return x diff --git a/vocoder/layers/pqmf2.py b/vocoder/layers/pqmf2.py new file mode 100644 index 00000000..4cffb819 --- /dev/null +++ b/vocoder/layers/pqmf2.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- + +# Copyright 2020 Tomoki Hayashi +# MIT License (https://opensource.org/licenses/MIT) + +"""Pseudo QMF modules.""" + +import numpy as np +import torch +import torch.nn.functional as F + +from scipy.signal import kaiser + + +def design_prototype_filter(taps=62, cutoff_ratio=0.15, beta=9.0): + """Design prototype filter for PQMF. + + This method is based on `A Kaiser window approach for the design of prototype + filters of cosine modulated filterbanks`_. + + Args: + taps (int): The number of filter taps. + cutoff_ratio (float): Cut-off frequency ratio. + beta (float): Beta coefficient for kaiser window. + + Returns: + ndarray: Impluse response of prototype filter (taps + 1,). + + .. _`A Kaiser window approach for the design of prototype filters of cosine modulated filterbanks`: + https://ieeexplore.ieee.org/abstract/document/681427 + + """ + # check the arguments are valid + assert taps % 2 == 0, "The number of taps mush be even number." + assert 0.0 < cutoff_ratio < 1.0, "Cutoff ratio must be > 0.0 and < 1.0." + + # make initial filter + omega_c = np.pi * cutoff_ratio + with np.errstate(invalid='ignore'): + h_i = np.sin(omega_c * (np.arange(taps + 1) - 0.5 * taps)) \ + / (np.pi * (np.arange(taps + 1) - 0.5 * taps)) + h_i[taps // 2] = np.cos(0) * cutoff_ratio # fix nan due to indeterminate form + + # apply kaiser window + w = kaiser(taps + 1, beta) + h = h_i * w + + return h + + +class PQMF(torch.nn.Module): + """PQMF module. + + This module is based on `Near-perfect-reconstruction pseudo-QMF banks`_. + + .. _`Near-perfect-reconstruction pseudo-QMF banks`: + https://ieeexplore.ieee.org/document/258122 + + """ + + def __init__(self, subbands=4, taps=62, cutoff_ratio=0.15, beta=9.0): + """Initilize PQMF module. + + Args: + subbands (int): The number of subbands. + taps (int): The number of filter taps. + cutoff_ratio (float): Cut-off frequency ratio. + beta (float): Beta coefficient for kaiser window. + + """ + super(PQMF, self).__init__() + + # define filter coefficient + h_proto = design_prototype_filter(taps, cutoff_ratio, beta) + h_analysis = np.zeros((subbands, len(h_proto))) + h_synthesis = np.zeros((subbands, len(h_proto))) + for k in range(subbands): + h_analysis[k] = 2 * h_proto * np.cos((2 * k + 1) * (np.pi / (2 * subbands)) * (np.arange(taps + 1) - ((taps - 1) / 2)) + (-1) ** k * np.pi / 4) + h_synthesis[k] = 2 * h_proto * np.cos((2 * k + 1) * (np.pi / (2 * subbands)) * (np.arange(taps + 1) - ((taps - 1) / 2)) - (-1) ** k * np.pi / 4) + + # convert to tensor + analysis_filter = torch.from_numpy(h_analysis).float().unsqueeze(1) + synthesis_filter = torch.from_numpy(h_synthesis).float().unsqueeze(0) + + # register coefficients as beffer + self.register_buffer("analysis_filter", analysis_filter) + self.register_buffer("synthesis_filter", synthesis_filter) + + # filter for downsampling & upsampling + updown_filter = torch.zeros((subbands, subbands, subbands)).float() + for k in range(subbands): + updown_filter[k, k, 0] = 1.0 + self.register_buffer("updown_filter", updown_filter) + self.subbands = subbands + + # keep padding info + self.pad_fn = torch.nn.ConstantPad1d(taps // 2, 0.0) + + def analysis(self, x): + """Analysis with PQMF. + + Args: + x (Tensor): Input tensor (B, 1, T). + + Returns: + Tensor: Output tensor (B, subbands, T // subbands). + + """ + x = F.conv1d(self.pad_fn(x), self.analysis_filter) + return F.conv1d(x, self.updown_filter, stride=self.subbands) + + def synthesis(self, x): + """Synthesis with PQMF. + + Args: + x (Tensor): Input tensor (B, subbands, T // subbands). + + Returns: + Tensor: Output tensor (B, 1, T). + + """ + # NOTE(kan-bayashi): Power will be dreased so here multipy by # subbands. + # Not sure this is the correct way, it is better to check again. + # TODO(kan-bayashi): Understand the reconstruction procedure + x = F.conv_transpose1d(x, self.updown_filter * self.subbands, stride=self.subbands) + x = F.conv1d(self.pad_fn(x), self.synthesis_filter) + return x diff --git a/vocoder/layers/qmf.dat b/vocoder/layers/qmf.dat new file mode 100644 index 00000000..17eab137 --- /dev/null +++ b/vocoder/layers/qmf.dat @@ -0,0 +1,640 @@ + 0.0000000e+000 + -5.5252865e-004 + -5.6176926e-004 + -4.9475181e-004 + -4.8752280e-004 + -4.8937912e-004 + -5.0407143e-004 + -5.2265643e-004 + -5.4665656e-004 + -5.6778026e-004 + -5.8709305e-004 + -6.1327474e-004 + -6.3124935e-004 + -6.5403334e-004 + -6.7776908e-004 + -6.9416146e-004 + -7.1577365e-004 + -7.2550431e-004 + -7.4409419e-004 + -7.4905981e-004 + -7.6813719e-004 + -7.7248486e-004 + -7.8343323e-004 + -7.7798695e-004 + -7.8036647e-004 + -7.8014496e-004 + -7.7579773e-004 + -7.6307936e-004 + -7.5300014e-004 + -7.3193572e-004 + -7.2153920e-004 + -6.9179375e-004 + -6.6504151e-004 + -6.3415949e-004 + -5.9461189e-004 + -5.5645764e-004 + -5.1455722e-004 + -4.6063255e-004 + -4.0951215e-004 + -3.5011759e-004 + -2.8969812e-004 + -2.0983373e-004 + -1.4463809e-004 + -6.1733441e-005 + 1.3494974e-005 + 1.0943831e-004 + 2.0430171e-004 + 2.9495311e-004 + 4.0265402e-004 + 5.1073885e-004 + 6.2393761e-004 + 7.4580259e-004 + 8.6084433e-004 + 9.8859883e-004 + 1.1250155e-003 + 1.2577885e-003 + 1.3902495e-003 + 1.5443220e-003 + 1.6868083e-003 + 1.8348265e-003 + 1.9841141e-003 + 2.1461584e-003 + 2.3017255e-003 + 2.4625617e-003 + 2.6201759e-003 + 2.7870464e-003 + 2.9469448e-003 + 3.1125421e-003 + 3.2739613e-003 + 3.4418874e-003 + 3.6008268e-003 + 3.7603923e-003 + 3.9207432e-003 + 4.0819753e-003 + 4.2264269e-003 + 4.3730720e-003 + 4.5209853e-003 + 4.6606461e-003 + 4.7932561e-003 + 4.9137604e-003 + 5.0393023e-003 + 5.1407354e-003 + 5.2461166e-003 + 5.3471681e-003 + 5.4196776e-003 + 5.4876040e-003 + 5.5475715e-003 + 5.5938023e-003 + 5.6220643e-003 + 5.6455197e-003 + 5.6389200e-003 + 5.6266114e-003 + 5.5917129e-003 + 5.5404364e-003 + 5.4753783e-003 + 5.3838976e-003 + 5.2715759e-003 + 5.1382275e-003 + 4.9839688e-003 + 4.8109469e-003 + 4.6039530e-003 + 4.3801862e-003 + 4.1251642e-003 + 3.8456408e-003 + 3.5401247e-003 + 3.2091886e-003 + 2.8446758e-003 + 2.4508540e-003 + 2.0274176e-003 + 1.5784683e-003 + 1.0902329e-003 + 5.8322642e-004 + 2.7604519e-005 + -5.4642809e-004 + -1.1568136e-003 + -1.8039473e-003 + -2.4826724e-003 + -3.1933778e-003 + -3.9401124e-003 + -4.7222596e-003 + -5.5337211e-003 + -6.3792293e-003 + -7.2615817e-003 + -8.1798233e-003 + -9.1325330e-003 + -1.0115022e-002 + -1.1131555e-002 + -1.2185000e-002 + -1.3271822e-002 + -1.4390467e-002 + -1.5540555e-002 + -1.6732471e-002 + -1.7943338e-002 + -1.9187243e-002 + -2.0453179e-002 + -2.1746755e-002 + -2.3068017e-002 + -2.4416099e-002 + -2.5787585e-002 + -2.7185943e-002 + -2.8607217e-002 + -3.0050266e-002 + -3.1501761e-002 + -3.2975408e-002 + -3.4462095e-002 + -3.5969756e-002 + -3.7481285e-002 + -3.9005368e-002 + -4.0534917e-002 + -4.2064909e-002 + -4.3609754e-002 + -4.5148841e-002 + -4.6684303e-002 + -4.8216572e-002 + -4.9738576e-002 + -5.1255616e-002 + -5.2763075e-002 + -5.4245277e-002 + -5.5717365e-002 + -5.7161645e-002 + -5.8591568e-002 + -5.9983748e-002 + -6.1345517e-002 + -6.2685781e-002 + -6.3971590e-002 + -6.5224711e-002 + -6.6436751e-002 + -6.7607599e-002 + -6.8704383e-002 + -6.9763024e-002 + -7.0762871e-002 + -7.1700267e-002 + -7.2568258e-002 + -7.3362026e-002 + -7.4100364e-002 + -7.4745256e-002 + -7.5313734e-002 + -7.5800836e-002 + -7.6199248e-002 + -7.6499217e-002 + -7.6709349e-002 + -7.6817398e-002 + -7.6823001e-002 + -7.6720492e-002 + -7.6505072e-002 + -7.6174832e-002 + -7.5730576e-002 + -7.5157626e-002 + -7.4466439e-002 + -7.3640601e-002 + -7.2677464e-002 + -7.1582636e-002 + -7.0353307e-002 + -6.8966401e-002 + -6.7452502e-002 + -6.5769067e-002 + -6.3944481e-002 + -6.1960278e-002 + -5.9816657e-002 + -5.7515269e-002 + -5.5046003e-002 + -5.2409382e-002 + -4.9597868e-002 + -4.6630331e-002 + -4.3476878e-002 + -4.0145828e-002 + -3.6641812e-002 + -3.2958393e-002 + -2.9082401e-002 + -2.5030756e-002 + -2.0799707e-002 + -1.6370126e-002 + -1.1762383e-002 + -6.9636862e-003 + -1.9765601e-003 + 3.2086897e-003 + 8.5711749e-003 + 1.4128883e-002 + 1.9883413e-002 + 2.5822729e-002 + 3.1953127e-002 + 3.8277657e-002 + 4.4780682e-002 + 5.1480418e-002 + 5.8370533e-002 + 6.5440985e-002 + 7.2694330e-002 + 8.0137293e-002 + 8.7754754e-002 + 9.5553335e-002 + 1.0353295e-001 + 1.1168269e-001 + 1.2000780e-001 + 1.2850029e-001 + 1.3715518e-001 + 1.4597665e-001 + 1.5496071e-001 + 1.6409589e-001 + 1.7338082e-001 + 1.8281725e-001 + 1.9239667e-001 + 2.0212502e-001 + 2.1197359e-001 + 2.2196527e-001 + 2.3206909e-001 + 2.4230169e-001 + 2.5264803e-001 + 2.6310533e-001 + 2.7366340e-001 + 2.8432142e-001 + 2.9507167e-001 + 3.0590986e-001 + 3.1682789e-001 + 3.2781137e-001 + 3.3887227e-001 + 3.4999141e-001 + 3.6115899e-001 + 3.7237955e-001 + 3.8363500e-001 + 3.9492118e-001 + 4.0623177e-001 + 4.1756969e-001 + 4.2891199e-001 + 4.4025538e-001 + 4.5159965e-001 + 4.6293081e-001 + 4.7424532e-001 + 4.8552531e-001 + 4.9677083e-001 + 5.0798175e-001 + 5.1912350e-001 + 5.3022409e-001 + 5.4125534e-001 + 5.5220513e-001 + 5.6307891e-001 + 5.7385241e-001 + 5.8454032e-001 + 5.9511231e-001 + 6.0557835e-001 + 6.1591099e-001 + 6.2612427e-001 + 6.3619801e-001 + 6.4612697e-001 + 6.5590163e-001 + 6.6551399e-001 + 6.7496632e-001 + 6.8423533e-001 + 6.9332824e-001 + 7.0223887e-001 + 7.1094104e-001 + 7.1944626e-001 + 7.2774489e-001 + 7.3582118e-001 + 7.4368279e-001 + 7.5131375e-001 + 7.5870808e-001 + 7.6586749e-001 + 7.7277809e-001 + 7.7942875e-001 + 7.8583531e-001 + 7.9197358e-001 + 7.9784664e-001 + 8.0344858e-001 + 8.0876950e-001 + 8.1381913e-001 + 8.1857760e-001 + 8.2304199e-001 + 8.2722753e-001 + 8.3110385e-001 + 8.3469374e-001 + 8.3797173e-001 + 8.4095414e-001 + 8.4362383e-001 + 8.4598185e-001 + 8.4803158e-001 + 8.4978052e-001 + 8.5119715e-001 + 8.5230470e-001 + 8.5310209e-001 + 8.5357206e-001 + 8.5373856e-001 + 8.5357206e-001 + 8.5310209e-001 + 8.5230470e-001 + 8.5119715e-001 + 8.4978052e-001 + 8.4803158e-001 + 8.4598185e-001 + 8.4362383e-001 + 8.4095414e-001 + 8.3797173e-001 + 8.3469374e-001 + 8.3110385e-001 + 8.2722753e-001 + 8.2304199e-001 + 8.1857760e-001 + 8.1381913e-001 + 8.0876950e-001 + 8.0344858e-001 + 7.9784664e-001 + 7.9197358e-001 + 7.8583531e-001 + 7.7942875e-001 + 7.7277809e-001 + 7.6586749e-001 + 7.5870808e-001 + 7.5131375e-001 + 7.4368279e-001 + 7.3582118e-001 + 7.2774489e-001 + 7.1944626e-001 + 7.1094104e-001 + 7.0223887e-001 + 6.9332824e-001 + 6.8423533e-001 + 6.7496632e-001 + 6.6551399e-001 + 6.5590163e-001 + 6.4612697e-001 + 6.3619801e-001 + 6.2612427e-001 + 6.1591099e-001 + 6.0557835e-001 + 5.9511231e-001 + 5.8454032e-001 + 5.7385241e-001 + 5.6307891e-001 + 5.5220513e-001 + 5.4125534e-001 + 5.3022409e-001 + 5.1912350e-001 + 5.0798175e-001 + 4.9677083e-001 + 4.8552531e-001 + 4.7424532e-001 + 4.6293081e-001 + 4.5159965e-001 + 4.4025538e-001 + 4.2891199e-001 + 4.1756969e-001 + 4.0623177e-001 + 3.9492118e-001 + 3.8363500e-001 + 3.7237955e-001 + 3.6115899e-001 + 3.4999141e-001 + 3.3887227e-001 + 3.2781137e-001 + 3.1682789e-001 + 3.0590986e-001 + 2.9507167e-001 + 2.8432142e-001 + 2.7366340e-001 + 2.6310533e-001 + 2.5264803e-001 + 2.4230169e-001 + 2.3206909e-001 + 2.2196527e-001 + 2.1197359e-001 + 2.0212502e-001 + 1.9239667e-001 + 1.8281725e-001 + 1.7338082e-001 + 1.6409589e-001 + 1.5496071e-001 + 1.4597665e-001 + 1.3715518e-001 + 1.2850029e-001 + 1.2000780e-001 + 1.1168269e-001 + 1.0353295e-001 + 9.5553335e-002 + 8.7754754e-002 + 8.0137293e-002 + 7.2694330e-002 + 6.5440985e-002 + 5.8370533e-002 + 5.1480418e-002 + 4.4780682e-002 + 3.8277657e-002 + 3.1953127e-002 + 2.5822729e-002 + 1.9883413e-002 + 1.4128883e-002 + 8.5711749e-003 + 3.2086897e-003 + -1.9765601e-003 + -6.9636862e-003 + -1.1762383e-002 + -1.6370126e-002 + -2.0799707e-002 + -2.5030756e-002 + -2.9082401e-002 + -3.2958393e-002 + -3.6641812e-002 + -4.0145828e-002 + -4.3476878e-002 + -4.6630331e-002 + -4.9597868e-002 + -5.2409382e-002 + -5.5046003e-002 + -5.7515269e-002 + -5.9816657e-002 + -6.1960278e-002 + -6.3944481e-002 + -6.5769067e-002 + -6.7452502e-002 + -6.8966401e-002 + -7.0353307e-002 + -7.1582636e-002 + -7.2677464e-002 + -7.3640601e-002 + -7.4466439e-002 + -7.5157626e-002 + -7.5730576e-002 + -7.6174832e-002 + -7.6505072e-002 + -7.6720492e-002 + -7.6823001e-002 + -7.6817398e-002 + -7.6709349e-002 + -7.6499217e-002 + -7.6199248e-002 + -7.5800836e-002 + -7.5313734e-002 + -7.4745256e-002 + -7.4100364e-002 + -7.3362026e-002 + -7.2568258e-002 + -7.1700267e-002 + -7.0762871e-002 + -6.9763024e-002 + -6.8704383e-002 + -6.7607599e-002 + -6.6436751e-002 + -6.5224711e-002 + -6.3971590e-002 + -6.2685781e-002 + -6.1345517e-002 + -5.9983748e-002 + -5.8591568e-002 + -5.7161645e-002 + -5.5717365e-002 + -5.4245277e-002 + -5.2763075e-002 + -5.1255616e-002 + -4.9738576e-002 + -4.8216572e-002 + -4.6684303e-002 + -4.5148841e-002 + -4.3609754e-002 + -4.2064909e-002 + -4.0534917e-002 + -3.9005368e-002 + -3.7481285e-002 + -3.5969756e-002 + -3.4462095e-002 + -3.2975408e-002 + -3.1501761e-002 + -3.0050266e-002 + -2.8607217e-002 + -2.7185943e-002 + -2.5787585e-002 + -2.4416099e-002 + -2.3068017e-002 + -2.1746755e-002 + -2.0453179e-002 + -1.9187243e-002 + -1.7943338e-002 + -1.6732471e-002 + -1.5540555e-002 + -1.4390467e-002 + -1.3271822e-002 + -1.2185000e-002 + -1.1131555e-002 + -1.0115022e-002 + -9.1325330e-003 + -8.1798233e-003 + -7.2615817e-003 + -6.3792293e-003 + -5.5337211e-003 + -4.7222596e-003 + -3.9401124e-003 + -3.1933778e-003 + -2.4826724e-003 + -1.8039473e-003 + -1.1568136e-003 + -5.4642809e-004 + 2.7604519e-005 + 5.8322642e-004 + 1.0902329e-003 + 1.5784683e-003 + 2.0274176e-003 + 2.4508540e-003 + 2.8446758e-003 + 3.2091886e-003 + 3.5401247e-003 + 3.8456408e-003 + 4.1251642e-003 + 4.3801862e-003 + 4.6039530e-003 + 4.8109469e-003 + 4.9839688e-003 + 5.1382275e-003 + 5.2715759e-003 + 5.3838976e-003 + 5.4753783e-003 + 5.5404364e-003 + 5.5917129e-003 + 5.6266114e-003 + 5.6389200e-003 + 5.6455197e-003 + 5.6220643e-003 + 5.5938023e-003 + 5.5475715e-003 + 5.4876040e-003 + 5.4196776e-003 + 5.3471681e-003 + 5.2461166e-003 + 5.1407354e-003 + 5.0393023e-003 + 4.9137604e-003 + 4.7932561e-003 + 4.6606461e-003 + 4.5209853e-003 + 4.3730720e-003 + 4.2264269e-003 + 4.0819753e-003 + 3.9207432e-003 + 3.7603923e-003 + 3.6008268e-003 + 3.4418874e-003 + 3.2739613e-003 + 3.1125421e-003 + 2.9469448e-003 + 2.7870464e-003 + 2.6201759e-003 + 2.4625617e-003 + 2.3017255e-003 + 2.1461584e-003 + 1.9841141e-003 + 1.8348265e-003 + 1.6868083e-003 + 1.5443220e-003 + 1.3902495e-003 + 1.2577885e-003 + 1.1250155e-003 + 9.8859883e-004 + 8.6084433e-004 + 7.4580259e-004 + 6.2393761e-004 + 5.1073885e-004 + 4.0265402e-004 + 2.9495311e-004 + 2.0430171e-004 + 1.0943831e-004 + 1.3494974e-005 + -6.1733441e-005 + -1.4463809e-004 + -2.0983373e-004 + -2.8969812e-004 + -3.5011759e-004 + -4.0951215e-004 + -4.6063255e-004 + -5.1455722e-004 + -5.5645764e-004 + -5.9461189e-004 + -6.3415949e-004 + -6.6504151e-004 + -6.9179375e-004 + -7.2153920e-004 + -7.3193572e-004 + -7.5300014e-004 + -7.6307936e-004 + -7.7579773e-004 + -7.8014496e-004 + -7.8036647e-004 + -7.7798695e-004 + -7.8343323e-004 + -7.7248486e-004 + -7.6813719e-004 + -7.4905981e-004 + -7.4409419e-004 + -7.2550431e-004 + -7.1577365e-004 + -6.9416146e-004 + -6.7776908e-004 + -6.5403334e-004 + -6.3124935e-004 + -6.1327474e-004 + -5.8709305e-004 + -5.6778026e-004 + -5.4665656e-004 + -5.2265643e-004 + -5.0407143e-004 + -4.8937912e-004 + -4.8752280e-004 + -4.9475181e-004 + -5.6176926e-004 + -5.5252865e-004 diff --git a/vocoder/models/__init__.py b/vocoder/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/models/melgan_discriminator.py b/vocoder/models/melgan_discriminator.py new file mode 100644 index 00000000..55d3f585 --- /dev/null +++ b/vocoder/models/melgan_discriminator.py @@ -0,0 +1,80 @@ +import numpy as np +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.utils import weight_norm + + +class MelganDiscriminator(nn.Module): + def __init__(self, + in_channels=1, + out_channels=1, + kernel_sizes=(5, 3), + base_channels=16, + max_channels=1024, + downsample_factors=(4, 4, 4, 4)): + super(MelganDiscriminator, self).__init__() + self.layers = nn.ModuleList() + + layer_kernel_size = np.prod(kernel_sizes) + layer_padding = (layer_kernel_size - 1) // 2 + + # initial layer + self.layers += [ + nn.Sequential( + nn.ReflectionPad1d(layer_padding), + weight_norm( + nn.Conv1d(in_channels, + base_channels, + layer_kernel_size, + stride=1)), nn.LeakyReLU(0.2, inplace=True)) + ] + + # downsampling layers + layer_in_channels = base_channels + for idx, downsample_factor in enumerate(downsample_factors): + layer_out_channels = min(layer_in_channels * downsample_factor, + max_channels) + layer_kernel_size = downsample_factor * 10 + 1 + layer_padding = (layer_kernel_size - 1) // 2 + layer_groups = layer_in_channels // 4 + self.layers += [ + nn.Sequential( + weight_norm( + nn.Conv1d(layer_in_channels, + layer_out_channels, + kernel_size=layer_kernel_size, + stride=downsample_factor, + padding=layer_padding, + groups=layer_groups)), + nn.LeakyReLU(0.2, inplace=True)) + ] + layer_in_channels = layer_out_channels + + # last 2 layers + layer_padding1 = (kernel_sizes[0] - 1) // 2 + layer_padding2 = (kernel_sizes[1] - 1) // 2 + self.layers += [ + nn.Sequential( + weight_norm( + nn.Conv1d(layer_out_channels, + layer_out_channels, + kernel_size=kernel_sizes[0], + stride=1, + padding=layer_padding1)), + nn.LeakyReLU(0.2, inplace=True), + ), + weight_norm( + nn.Conv1d(layer_out_channels, + out_channels, + kernel_size=kernel_sizes[1], + stride=1, + padding=layer_padding2)), + ] + + def forward(self, x): + feats = [] + for layer in self.layers: + x = layer(x) + feats.append(x) + return x, feats diff --git a/vocoder/models/melgan_generator.py b/vocoder/models/melgan_generator.py new file mode 100644 index 00000000..2d266f29 --- /dev/null +++ b/vocoder/models/melgan_generator.py @@ -0,0 +1,91 @@ +import math +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.utils import weight_norm + +from TTS.vocoder.layers.melgan import ResidualStack + + +class MelganGenerator(nn.Module): + def __init__(self, + in_channels=80, + out_channels=1, + proj_kernel=7, + base_channels=512, + upsample_factors=(8, 8, 2, 2), + res_kernel=3, + num_res_blocks=3): + super(MelganGenerator, self).__init__() + + # assert model parameters + assert (proj_kernel - + 1) % 2 == 0, " [!] proj_kernel should be an odd number." + + # setup additional model parameters + base_padding = (proj_kernel - 1) // 2 + act_slope = 0.2 + self.inference_padding = 2 + + # initial layer + layers = [] + layers += [ + nn.ReflectionPad1d(base_padding), + weight_norm( + nn.Conv1d(in_channels, + base_channels, + kernel_size=proj_kernel, + stride=1, + bias=True)) + ] + + # upsampling layers and residual stacks + for idx, upsample_factor in enumerate(upsample_factors): + layer_in_channels = base_channels // (2**idx) + layer_out_channels = base_channels // (2**(idx + 1)) + layer_filter_size = upsample_factor * 2 + layer_stride = upsample_factor + layer_output_padding = upsample_factor % 2 + layer_padding = upsample_factor // 2 + layer_output_padding + layers += [ + nn.LeakyReLU(act_slope), + weight_norm( + nn.ConvTranspose1d(layer_in_channels, + layer_out_channels, + layer_filter_size, + stride=layer_stride, + padding=layer_padding, + output_padding=layer_output_padding, + bias=True)), + ResidualStack( + channels=layer_out_channels, + num_res_blocks=num_res_blocks, + kernel_size=res_kernel + ) + ] + + layers += [nn.LeakyReLU(act_slope)] + + # final layer + layers += [ + nn.ReflectionPad1d(base_padding), + weight_norm( + nn.Conv1d(layer_out_channels, + out_channels, + proj_kernel, + stride=1, + bias=True)), + nn.Tanh() + ] + self.layers = nn.Sequential(*layers) + + def forward(self, cond_features): + return self.layers(cond_features) + + def inference(self, cond_features): + cond_features = cond_features.to(self.layers[1].weight.device) + cond_features = torch.nn.functional.pad( + cond_features, + (self.inference_padding, self.inference_padding), + 'replicate') + return self.layers(cond_features) diff --git a/vocoder/models/melgan_multiscale_discriminator.py b/vocoder/models/melgan_multiscale_discriminator.py new file mode 100644 index 00000000..d77b9ceb --- /dev/null +++ b/vocoder/models/melgan_multiscale_discriminator.py @@ -0,0 +1,41 @@ +from torch import nn + +from TTS.vocoder.models.melgan_discriminator import MelganDiscriminator + + +class MelganMultiscaleDiscriminator(nn.Module): + def __init__(self, + in_channels=1, + out_channels=1, + num_scales=3, + kernel_sizes=(5, 3), + base_channels=16, + max_channels=1024, + downsample_factors=(4, 4, 4, 4), + pooling_kernel_size=4, + pooling_stride=2, + pooling_padding=1): + super(MelganMultiscaleDiscriminator, self).__init__() + + self.discriminators = nn.ModuleList([ + MelganDiscriminator(in_channels=in_channels, + out_channels=out_channels, + kernel_sizes=kernel_sizes, + base_channels=base_channels, + max_channels=max_channels, + downsample_factors=downsample_factors) + for _ in range(num_scales) + ]) + + self.pooling = nn.AvgPool1d(kernel_size=pooling_kernel_size, stride=pooling_stride, padding=pooling_padding, count_include_pad=False) + + + def forward(self, x): + scores = list() + feats = list() + for disc in self.discriminators: + score, feat = disc(x) + scores.append(score) + feats.append(feat) + x = self.pooling(x) + return scores, feats \ No newline at end of file diff --git a/vocoder/models/multiband_melgan_generator.py b/vocoder/models/multiband_melgan_generator.py new file mode 100644 index 00000000..8feacd25 --- /dev/null +++ b/vocoder/models/multiband_melgan_generator.py @@ -0,0 +1,38 @@ +import torch + +from TTS.vocoder.models.melgan_generator import MelganGenerator +from TTS.vocoder.layers.pqmf import PQMF + + +class MultibandMelganGenerator(MelganGenerator): + def __init__(self, + in_channels=80, + out_channels=4, + proj_kernel=7, + base_channels=384, + upsample_factors=(2, 8, 2, 2), + res_kernel=3, + num_res_blocks=3): + super(MultibandMelganGenerator, + self).__init__(in_channels=in_channels, + out_channels=out_channels, + proj_kernel=proj_kernel, + base_channels=base_channels, + upsample_factors=upsample_factors, + res_kernel=res_kernel, + num_res_blocks=num_res_blocks) + self.pqmf_layer = PQMF(N=4, taps=62, cutoff=0.15, beta=9.0) + + def pqmf_analysis(self, x): + return self.pqmf_layer.analysis(x) + + def pqmf_synthesis(self, x): + return self.pqmf_layer.synthesis(x) + + def inference(self, cond_features): + cond_features = cond_features.to(self.layers[1].weight.device) + cond_features = torch.nn.functional.pad( + cond_features, + (self.inference_padding, self.inference_padding), + 'replicate') + return self.pqmf.synthesis(self.layers(cond_features)) diff --git a/vocoder/models/random_window_discriminator.py b/vocoder/models/random_window_discriminator.py new file mode 100644 index 00000000..3efd395e --- /dev/null +++ b/vocoder/models/random_window_discriminator.py @@ -0,0 +1,225 @@ +import numpy as np +from torch import nn + + +class GBlock(nn.Module): + def __init__(self, in_channels, cond_channels, downsample_factor): + super(GBlock, self).__init__() + + self.in_channels = in_channels + self.cond_channels = cond_channels + self.downsample_factor = downsample_factor + + self.start = nn.Sequential( + nn.AvgPool1d(downsample_factor, stride=downsample_factor), + nn.ReLU(), + nn.Conv1d(in_channels, in_channels * 2, kernel_size=3, padding=1)) + self.lc_conv1d = nn.Conv1d(cond_channels, + in_channels * 2, + kernel_size=1) + self.end = nn.Sequential( + nn.ReLU(), + nn.Conv1d(in_channels * 2, + in_channels * 2, + kernel_size=3, + dilation=2, + padding=2)) + self.residual = nn.Sequential( + nn.Conv1d(in_channels, in_channels * 2, kernel_size=1), + nn.AvgPool1d(downsample_factor, stride=downsample_factor)) + + def forward(self, inputs, conditions): + outputs = self.start(inputs) + self.lc_conv1d(conditions) + outputs = self.end(outputs) + residual_outputs = self.residual(inputs) + outputs = outputs + residual_outputs + + return outputs + + +class DBlock(nn.Module): + def __init__(self, in_channels, out_channels, downsample_factor): + super(DBlock, self).__init__() + + self.in_channels = in_channels + self.downsample_factor = downsample_factor + self.out_channels = out_channels + + self.donwsample_layer = nn.AvgPool1d(downsample_factor, + stride=downsample_factor) + self.layers = nn.Sequential( + nn.ReLU(), + nn.Conv1d(in_channels, out_channels, kernel_size=3, padding=1), + nn.ReLU(), + nn.Conv1d(out_channels, + out_channels, + kernel_size=3, + dilation=2, + padding=2)) + self.residual = nn.Sequential( + nn.Conv1d(in_channels, out_channels, kernel_size=1), ) + + def forward(self, inputs): + if self.downsample_factor > 1: + outputs = self.layers(self.donwsample_layer(inputs))\ + + self.donwsample_layer(self.residual(inputs)) + else: + outputs = self.layers(inputs) + self.residual(inputs) + return outputs + + +class ConditionalDiscriminator(nn.Module): + def __init__(self, + in_channels, + cond_channels, + downsample_factors=(2, 2, 2), + out_channels=(128, 256)): + super(ConditionalDiscriminator, self).__init__() + + assert len(downsample_factors) == len(out_channels) + 1 + + self.in_channels = in_channels + self.cond_channels = cond_channels + self.downsample_factors = downsample_factors + self.out_channels = out_channels + + self.pre_cond_layers = nn.ModuleList() + self.post_cond_layers = nn.ModuleList() + + # layers before condition features + self.pre_cond_layers += [DBlock(in_channels, 64, 1)] + in_channels = 64 + for (i, channel) in enumerate(out_channels): + self.pre_cond_layers.append( + DBlock(in_channels, channel, downsample_factors[i])) + in_channels = channel + + # condition block + self.cond_block = GBlock(in_channels, cond_channels, + downsample_factors[-1]) + + # layers after condition block + self.post_cond_layers += [ + DBlock(in_channels * 2, in_channels * 2, 1), + DBlock(in_channels * 2, in_channels * 2, 1), + nn.AdaptiveAvgPool1d(1), + nn.Conv1d(in_channels * 2, 1, kernel_size=1), + ] + + def forward(self, inputs, conditions): + batch_size = inputs.size()[0] + outputs = inputs.view(batch_size, self.in_channels, -1) + for layer in self.pre_cond_layers: + outputs = layer(outputs) + outputs = self.cond_block(outputs, conditions) + for layer in self.post_cond_layers: + outputs = layer(outputs) + + return outputs + + +class UnconditionalDiscriminator(nn.Module): + def __init__(self, + in_channels, + base_channels=64, + downsample_factors=(8, 4), + out_channels=(128, 256)): + super(UnconditionalDiscriminator, self).__init__() + + self.downsample_factors = downsample_factors + self.in_channels = in_channels + self.downsample_factors = downsample_factors + self.out_channels = out_channels + + self.layers = nn.ModuleList() + self.layers += [DBlock(self.in_channels, base_channels, 1)] + in_channels = base_channels + for (i, factor) in enumerate(downsample_factors): + self.layers.append(DBlock(in_channels, out_channels[i], factor)) + in_channels *= 2 + self.layers += [ + DBlock(in_channels, in_channels, 1), + DBlock(in_channels, in_channels, 1), + nn.AdaptiveAvgPool1d(1), + nn.Conv1d(in_channels, 1, kernel_size=1), + ] + + def forward(self, inputs): + batch_size = inputs.size()[0] + outputs = inputs.view(batch_size, self.in_channels, -1) + for layer in self.layers: + outputs = layer(outputs) + return outputs + + +class RandomWindowDiscriminator(nn.Module): + """Random Window Discriminator as described in + http://arxiv.org/abs/1909.11646""" + def __init__(self, + cond_channels, + hop_length, + uncond_disc_donwsample_factors=(8, 4), + cond_disc_downsample_factors=((8, 4, 2, 2, 2), (8, 4, 2, 2), + (8, 4, 2), (8, 4), (4, 2, 2)), + cond_disc_out_channels=((128, 128, 256, 256), (128, 256, 256), + (128, 256), (256, ), (128, 256)), + window_sizes=(512, 1024, 2048, 4096, 8192)): + + super(RandomWindowDiscriminator, self).__init__() + self.cond_channels = cond_channels + self.window_sizes = window_sizes + self.hop_length = hop_length + self.base_window_size = self.hop_length * 2 + self.ks = [ws // self.base_window_size for ws in window_sizes] + + # check arguments + assert len(cond_disc_downsample_factors) == len( + cond_disc_out_channels) == len(window_sizes) + for ws in window_sizes: + assert ws % hop_length == 0 + + for idx, cf in enumerate(cond_disc_downsample_factors): + assert np.prod(cf) == hop_length // self.ks[idx] + + # define layers + self.unconditional_discriminators = nn.ModuleList([]) + for k in self.ks: + layer = UnconditionalDiscriminator( + in_channels=k, + base_channels=64, + downsample_factors=uncond_disc_donwsample_factors) + self.unconditional_discriminators.append(layer) + + self.conditional_discriminators = nn.ModuleList([]) + for idx, k in enumerate(self.ks): + layer = ConditionalDiscriminator( + in_channels=k, + cond_channels=cond_channels, + downsample_factors=cond_disc_downsample_factors[idx], + out_channels=cond_disc_out_channels[idx]) + self.conditional_discriminators.append(layer) + + def forward(self, x, c): + scores = [] + feats = [] + # unconditional pass + for (window_size, layer) in zip(self.window_sizes, + self.unconditional_discriminators): + index = np.random.randint(x.shape[-1] - window_size) + + score = layer(x[:, :, index:index + window_size]) + scores.append(score) + + # conditional pass + for (window_size, layer) in zip(self.window_sizes, + self.conditional_discriminators): + frame_size = window_size // self.hop_length + lc_index = np.random.randint(c.shape[-1] - frame_size) + sample_index = lc_index * self.hop_length + x_sub = x[:, :, + sample_index:(lc_index + frame_size) * self.hop_length] + c_sub = c[:, :, lc_index:lc_index + frame_size] + + score = layer(x_sub, c_sub) + scores.append(score) + return scores, feats diff --git a/vocoder/notebooks/Untitled.ipynb b/vocoder/notebooks/Untitled.ipynb new file mode 100644 index 00000000..ce49d6fa --- /dev/null +++ b/vocoder/notebooks/Untitled.ipynb @@ -0,0 +1,678 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "#function example with several unknowns (variables) for optimization\n", + "#Gerald Schuller, Nov. 2016\n", + "import numpy as np\n", + "\n", + "def functionexamp(x):\n", + " #x: array with 2 variables\n", + " \n", + " y=np.sin(x[0])+np.cos(x[1])\n", + " return y" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " fun: -1.9999999999888387\n", + " jac: array([4.7236681e-06, 0.0000000e+00])\n", + " message: 'Optimization terminated successfully.'\n", + " nfev: 12\n", + " nit: 2\n", + " njev: 3\n", + " status: 0\n", + " success: True\n", + " x: array([-1.5707916 , -3.14159265])\n" + ] + } + ], + "source": [ + "#Optimization example, see also:\n", + "#https://docs.scipy.org/doc/scipy-0.18.1/reference/optimize.html\n", + "#Gerald Schuller, Nov. 2016\n", + "#run it with \"python optimizationExample.py\" in a termina shell\n", + "#or type \"ipython\" in a termina shell and copy lines below:\n", + "\n", + "import numpy as np\n", + "import scipy.optimize as optimize\n", + "\n", + "#Example for 2 unknowns, args: function-name, starting point, method:\n", + "xmin = optimize.minimize(functionexamp, [-1.0, -3.0], method='CG')\n", + "print(xmin)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "function [p,passedge] = opt_filter(filtorder,N)\n", + "\n", + "% opt_filter Create Lowpass Prototype Filter for the Pseudo-QMF \n", + "% Filter Bank with N Subbands\n", + "%\n", + "% Adapted from the paper by C. D. Creusere and S. K. Mitra, titled \n", + "% \"A simple method for designing high-quality prototype filters for \n", + "% M-band pseudo-QMF banks,\" IEEE Trans. Signal Processing,vol. 43, \n", + "% pp. 1005-1007, Apr. 1995 and the book by S. K. Mitra titled \"\n", + "% Digital Signal Processing: A Computer-Based Approach, McGraw-Hill, 2001\n", + "%\n", + "% Arguments:\n", + "% filtorder Filter order (i.e., filter length - 1)\n", + "% N Number of subbands\n", + "\n", + "stopedge = 1/N; % Stopband edge fixed at (1/N)pi\n", + "passedge = 1/(4*N); % Start value for passband edge\n", + "tol = 0.000001; % Tolerance\n", + "step = 0.1*passedge; % Step size for searching the passband edge\n", + "way = -1; % Search direction, increase or reduce the passband edge\n", + "tcost = 0; % Current error calculated with the cost function\n", + "pcost = 10; % Previous error calculated with the cost function\n", + "flag = 0; % Set to 1 to stop the search\n", + "\n", + "while flag == 0\n", + " \n", + "% Design the lowpass filter using Parks-McClellan algorithm\n", + " \n", + " p = remez(filtorder,[0,passedge,stopedge,1],[1,1,0,0],[5,1]);\n", + " \n", + "% Calculates the cost function according to Eq. (2.36)\n", + "\n", + " P = fft(p,4096);\n", + " OptRange = floor(2048/N); % 0 to pi/N\n", + " phi = zeros(OptRange,1); % Initialize to zeros\n", + "\n", + "% Compute the flatness in the range from 0 to pi/N\n", + "\n", + "\tfor k = 1:OptRange\n", + " phi(k) = abs(P(OptRange-k+2))^2 + abs(P(k))^2;\n", + "\tend\n", + "\ttcost = max(abs(phi - ones(max(size(phi)),1)));\n", + " \t\n", + "\tif tcost > pcost % If search in wrong direction\n", + "\t\tstep = step/2; % Reduce step size by half \n", + "\t\tway = -way; % Change the search direction \n", + "\tend\n", + "\t\n", + "\tif abs(pcost - tcost) < tol % If improvement is below tol \n", + "\t\tflag = 1; % Stop the search \n", + "\tend\n", + "\t\n", + "\tpcost = tcost;\n", + "\tpassedge = passedge + way*step; % Adjust the passband edge\n", + " \n", + "end" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "sig.remez" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0125" + ] + }, + "execution_count": 101, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "1 / 4. / 20.0" + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Band edges should be less than 1/2 the sampling frequency", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mremez\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m16.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m4.0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/scipy/signal/fir_filter_design.py\u001b[0m in \u001b[0;36mremez\u001b[0;34m(numtaps, bands, desired, weight, Hz, type, maxiter, grid_density, fs)\u001b[0m\n\u001b[1;32m 854\u001b[0m \u001b[0mbands\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbands\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 855\u001b[0m return sigtools._remez(numtaps, bands, desired, weight, tnum, fs,\n\u001b[0;32m--> 856\u001b[0;31m maxiter, grid_density)\n\u001b[0m\u001b[1;32m 857\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 858\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Band edges should be less than 1/2 the sampling frequency" + ] + } + ], + "source": [ + "p = sig.remez(65, [0, 1/16.0, 1/4.0, 1], [1, 0], [5, 1])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "def create_pqmf_filter(filter_len=64, N=4):\n", + " stop_edge = 1 / N\n", + " pass_edge = 1 / (4 * N)\n", + " tol = 1e-8\n", + " cutoff = 0.1 * pass_edge\n", + " cost = 0\n", + " cost_prev = float('inf')\n", + " \n", + " p = sig.remez(filter_len, [0, pass_edge, stop_edge, 1], [1, 1, 0, 0], [5, 1])\n", + " \n", + " P = sig.freqz(p, workN=2048)\n", + " opt_range = 2048 // N\n", + " phi = np.zeros(opt_range)\n", + " \n", + " H = np.abs(P)\n", + " phi = H[opt_range + 2] \n", + " for i in range(opt_range):\n", + " phi[i] = abs(P(opt_range - i + 2)) ** 2 + abs(P[i]) ** 2" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import scipy as sp\n", + "import scipy.signal as sig\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "\n", + "def optimfuncQMF(x):\n", + " \"\"\"Optimization function for a PQMF Filterbank\n", + " x: coefficients to optimize (first half of prototype h because of symmetry)\n", + " err: resulting total error\n", + " \"\"\"\n", + " K = ntaps * N \n", + " h = np.append(x, np.flipud(x))\n", + " cutoff = 0.15\n", + " \n", + "# breakpoint()\n", + " f, H_im = sig.freqz(h, worN=K)\n", + " H = np.abs(H_im) #only keeping the real part\n", + " \n", + " posfreq = np.square(H[0:K//N])\n", + " \n", + " #Negative frequencies are symmetric around 0:\n", + " negfreq = np.flipud(np.square(H[0:K//N]))\n", + " \n", + " #Sum of magnitude squared frequency responses should be closed to unity (or N)\n", + " unitycond = np.sum(np.abs(posfreq + negfreq - 2*(N*N)*np.ones(K//N)))/K\n", + " \n", + " #plt.plot(posfreq+negfreq)\n", + " \n", + " #High attenuation after the next subband:\n", + " att = np.sum(np.abs(H[int(cutoff*K//N):]))/K\n", + " \n", + " #Total (weighted) error:\n", + " err = unitycond + 100*att\n", + " return err" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(32,)" + ] + }, + "execution_count": 85, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xmin.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "8.684549400499243\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nO3dd3xUVfr48c+TRgglhSQQEkLvHQMIoqCAYEVX17Wsoqvr6vbid7/6dVdd19113aJbv/74KnaxF0SFRQTpJbQQauikkUYSEtLz/P6YC44xIT035Xm/XvPK3Dvnzjz3zmSeOefce46oKsYYY0x9+bgdgDHGmLbJEogxxpgGsQRijDGmQSyBGGOMaRBLIMYYYxrEEogxxpgGsQRiWpyIrBKRe87z+LMi8uuWjKmtEJFYESkQEd8Gbn9URGY1dVytiYjsFpEZbsfREVgC6eCcL5Qi50spXUReFJGuLfj6d4rIWu91qnqfqv62GV6rn4ios68Fzr4/6PW4iMh/iUiSc0yOi8jvRSTAq8yLznPMq/LcTzvr7/Tarwqv1yoQkX9WE9MtIrK3yrrlNax7UFWPq2pXVa1oosPSpETkMREpq7Lfv2zG13tRRJ7wXqeqI1V1VXO9pvmSJRADcI2qdgXGAeOBh1yOp7mFOPt7C/CIiMx11v8duBe4A+gGXAHMAt6osv0BpwwAIuIH3AQcqlJug/Nlf/b2w2piWQ0ME5EIr+caC3Susm6KU7YteLPKfj/ldkCmeVgCMeeoajqwDE8iAUBELhSR9SKSKyI7vZsGnF/Zh0XktIgcEZHbnPWPicirXuXO/vL38349ERkOPAtMcX6p5jrrz/2qFJEZIpIsIr8QkQwRSRORu7yeo4eIfCQi+SKyRUSeqFqjOc/+bgB2A6NEZDDwfeA2Vd2gquWquhu4AbhKRKZ7bfoRME1EQp3luUACkF6X160SQwpwGLjEWTXBiemLKut8gC1Vj6XTHPhbEVnnvA//EZFwr+Nzu4gcE5FsEXnY+7VFpJOIPCMiqc7tGRHp5Dz2hYjc4Ny/yHnNq5zlmSKyoz77Wdtnog77Mc3rc3jC+ezdC9wG/NL5/HzklD3XTFfLPp73s2VqZwnEnCMiMXh+dR90lqOBj4EngDDgAeBdEYkQkS54frFfoardgKlAvb5UVHUvcB9f/lIPqaFoLyAYiAbuBv7l9eX9L6DQKTPfudVlX0VELgJGAtuBmUCyqm6uEuMJYCNwudfqYuBD4GZn+Q7g5bq8bg1W82WyuARYA6ytsm6jqpbVsP2twF1AJBCA531CREYA/wvcDvQGegAxXts9DFyI5wfDWGAS8CvnsS+AGc796Xw1yU13Hm9qNe1HX+BT4B9AhBPvDlVdALwGPOV8fq6p5jnPt49w/s+WqYUlEAPwgYicBk4AGcCjzvpvA5+o6ieqWqmqy4F44Ern8Uo8v947q2qa84u9OZQBj6tqmap+AhQAQ8XTkXwD8KiqnlHVPcBLdXi+LCAHeA54UFVXAOFAWg3l0/B8cXl7GbhDRELwfKF+UM12Fzq/mM/eLqzh+b1rGxfjSSBrqqw73xf2C6p6QFWLgLf4sgZ5I7BEVVeragnwazzv2Vm34TmuGaqaCfwGT7I5G9PZWtclwB+8lmtLIDdV2e/e5ylbl/24FfhMVRc5n4FsVa3rj5Xz7SPU8Nmq43N3eJZADMB1Ti1iBjAMz5cpQF/gm95fBsA0IEpVC4Fv4alBpInIxyIyrJniy1bVcq/lM0BXPF/qfngS31ne92sSrqqhqjpcVf/urMsComooH+U8fo6qrnVe/2E8X9JF1Wy3UVVDvG4ba3j+1cAY55fvhXhqZPuAKGfdNM7f/+HddHb22ICn1nHueDjvWbZX2d7AMa/lY846gA3AEBHpieeL/GWgj9OsNKmWeN6qst+p5ylbl/3ow9f7l+rqfPsINX+2TB1YAjHnqOoXwIvAn51VJ4BXqnwZdFHVJ53yy1R1Np4v2H3A/znbFQJBXk/d63wv24iQM4Fyvtos06eBz/U5ni/ISd4rRaQPni/1VdVs8yrwCxrXfIWqHgZS8XTgH1fVAuehDc66rnia0eorDa/jISJBeJqxzkrF8yPhrFhnHap6BtgK/ARIVNVSYD3wc+CQqn4lodZBfT4TVZ0ABtbwWG2fnxr30TSeJRBT1TPAbBEZi+cL8hoRmSMiviIS6HQ8xohITxGZ5/SFlOCp+p9tHtkBXCKeaxaCOf9ZXSeBGPE6VbaunFNZ3wMeE5EgpwZ0Ry2b1fRcB/B06L8mnhMHfEVkJPAuni/Oz6rZ7O/AbJrm7Kg1eL6c13itW+usi6+hhlObd4CrnQ7oAOBxvvo/vwj4ldOnFQ48guc9P+sL4Id82Vy1qspyfdTnM1HVa8AsEblJRPzEc+LE2eatk8CA82xb2z6aRrAEYr7CaSd+GXjE6UCeB/wPnl/7J4D/wvO58cHz5ZaKpz9hOnC/8xzLgTfxnJm0FVhynpf8HM9ZR+kiUt9fteD5QgvG0/zxCp4vjJIGPM/Z53oOzxfMGSART5PHdapaWbWwquao6gptmkl1vsDTeex9BtkaZ12DEpTTJ/UD4HU8tZFTQLJXkSfw9GklALuAbc4675i6eb1+1eX6xFKfz0TVbY/j6Xf7BZ7P2g48HeIAzwMjnCbW6vqhattH0whiE0qZ9kRE/gj0UtU6nY1Vy3P9BrgeuERVcxsdnDHtjNVATJsmIsNEZIxzWu4kPKdivt8Uz62qjwIL8PSBGGOqsBqIadNEZCKeZqveeNrDFwBPNlGzkjHmPCyBGGOMaRBrwjLGGNMgfrUXaT/Cw8O1X79+bodhjDFtytatW7NUtepoDB0rgfTr14/4+Hi3wzDGmDZFRI5Vt96asIwxxjSIJRBjjDENYgnEGGNMg1gCMcYY0yCWQIwxxjSIJRBjjDENYgnEGGNMg1gCaSGf7krjUGZB7QWNMcZRWaks2nycnMJSt0OpliWQesg8XcI/ViRRWFJee2Ev6w5mcf9r23hscXNNGW6MaY9WJ2Xy0Hu7eODtndR33MLP9pxk2e702gs2giWQevj1B4n8ZfkB/rXyYJ23KSgp55fvJOAjsCYpi+PZZ5oxQmNMe7Jo83F8BD7fl8G721LqvF1KbhE/XLSNH72+vVlbPiyB1NHn+06ydHc6Ed068dyaI3VOBH/4ZC+peUX845YJ+Ai8seV4M0dqjGkPMvKL+WxvBndP68/EfqH85qPdpOcV12nbJz/dhyp08vfh1x8k1rv2UleWQOqgqLSCRz7czaDIrrx3/1T8fIXffbLnvNsUl1Xwt8+SeG3Tce6+qD9XjYnismE9eSs+mbKKr82OiqqycO0RLvvLKo5mFTbXrhhjWomH39/Fbc9tJON09Unh7a3JVFQqt0yK5akbx1JWUcm9r8SzP/30eZ9385EcPtqZyvemD+SXc4ex/lA2i3emNscuWAKpi398nkTyqSKeuG4UfcKC+MGlg1i2+ySrD2R+rWxeURkfbE9h7jOrefqzA1w1JooH5gwF4NbJfcgqKGHF3pNf2aasopL/eT+Rx5fs4XBmIb/+8Ku/GMqrSTjGmLbF+/941f4MXtt0nHUHs7nun+vYk5r/lbJnO8+nDOjBgIiu9A/vwtM3jeN4zhmu/PsaHv9oD/vS879WsyirqOTxJbuJCg7k/ukDuXVSLGNjgvntkj3kFZU1+T65OhqviMwF/gb4As+p6pNVHu8EvAxcAGQD31LVo85jD+GZvrQC+LGqLmuuOCsqlZviYrhwQA8A7p7Wn3e2JvP917ax4I4LmDownMSUPJ5atp/1B7Mor1QGhHfh5e9M4pIhX46APH1IJFHBgfzj84McyiykslLZd/I0246dIi2vmO/PGEh41048vmQPSxLSmD40gh8v2s7BjAL+87NLCAqo/e3al57PvrTTXDc+urkOhzEd3omcM6xJyuLWybF1Kv/vVQd5dtUh/nbLeKYM6MEjH+5mQEQX/vzNsXz/1W1c9+91jOsTwtiYYEK7BJCRX0LyqSJ+OXfYuee4YnQUkwf04E/L9vHC+iMsXHeEmNDO3DGlL3dd1J/yCuX+17aSmJLPv26dQOcAXwB+d/1o/uudBLIKSgju7N+kx8G1GQlFxBc4AMwGkoEtwC2quserzPeBMap6n4jcDFyvqt8SkRF4pjGdhGcq08+AIapacb7XjIuL04YO566qiMi55fS8Ym5/fhPHss8wZ1QvPk5IJaxLAN+M68Os4ZGM6xOKr4987XmeW3OYJz7ee245OqQzY/sEc+3Y3swdFUVFpXLdv9aRnl9McGd/jmQVUlGp/M+Vw7j3koG1xnn785tYfyib7Y/Mpntg035YjDEeD723i0Wbj7P0pxczrFf385bNO1PGtD9+TlFZBZWqTOofxsbDObx+z2SmDgonI7+Y//3iENuP57InLZ/Sck9NJTYsiOU/v4ROfr5fe86M/GJW7MtgSUIq6w5mM6RnV4IC/EhIzuV314/mlklfTWxVv7/qS0S2qmpc1fVu1kAmAQdV9TCAiLwBzAO8OxfmAY85998B/imeozAPeENVS4AjInLQeb4NzRVs1YPfKziQt++bwl0vbmFJQiq3X9iXX1w+tNYMf8/FA7hzaj8qVFGFQP+vfjh8fYTfXT+Kef9aR2l5Ja/ePZl/rzrIgtWHuf3Cfud+VVQn43Qx6w5mUamw4VA2c0b2avgOG2NqtPagp/n6wx2pDJt7/gTy/LojnC4p5937p/LsF4dYvuck14+PZuqgcAAiuwfy6DUjAU8zV3ml4usj+PlIjV/6kd0DuWVSLLdMimX5npM8tng3R7PO8K9bJ3DF6KivlW9M8jgfNxNINHDCazkZmFxTGVUtF5E8oIezfmOVbVu8zSYkKIA3751CZkEJ0SGd67ydn6/PeQ/8mJgQXr/nQmJCO9MnLAh/X+HGZzfw2qZj3HPxgHPlSsorOJhRwMjewQB8nJBGpYK/r7DuYJYlEGOawfHsM5zIKcLfV1i8I5X/unwoPj5CVkEJRaUV9AkLOlc2r6iMF9YdYc7InlzQN5T/9+0L+M+ek0wbHF7tc/v5+lBNheO8Zo/oycWDwzl1ppSo4Lp/DzWFdt+JLiL3iki8iMRnZn6907uxAvx86pU86mrKwB7nPohx/cK4aFAPnv3i8LmLGHPPlHL785u56u9rWZLgOcPiwx2pjIjqzrRB4axNymrymIwxsMapfXzvkoGk5Bax7fgpikor+OazG5j99Bf8x+vivefXHuF0cTk/njkYAB8fYe6oXnTt1LS/3QP9fVs8eYC7NZAUoI/XcoyzrroyySLiBwTj6Uyvy7YAqOoCYAF4+kCaJHIX/GTmEG76fxuY/qeV3DIplk92pXEip4i+PYL49QeJ9OoeyI4TuTx0xTD8fH1YuX8PKblFzZLcjOnI1iZlERUcyH0zBvLc2sN8sCOFj3elcSSrkEGRXfneq1v53iUD2Xkilw2Hs5kzsue5VoL2xs0ayBZgsIj0F5EA4GZgcZUyi4H5zv0bgc/V0+u/GLhZRDqJSH9gMLC5heJ2xaT+YSz67oWMiQnhH58fJKuglFfunsTz8+MoLK3gzhe2IALXjuvNxU71eG1S09e4jOnIKiqV9YeymTYonK6d/Jg1vCfvbUvhhXVHmT+lLx/9cBqzhvfk2S8OcTS7kAevGMZfbhrndtjNxrUaiNOn8UNgGZ7TeBeq6m4ReRyIV9XFwPPAK04neQ6eJINT7i08He7lwA9qOwOrPZgysAdTBvbgWHYhAX4+56qsD1w+hN9/so/J/cOICu5Mr+5KZLdOrD2Yzbcm1u00Q2NM7RJT8sgrKjvXhzFvXDRLEtLo1yOI/75iGJ0DfHn22xeQkJzLqOhg/H3bdy+Bq9eBqOonwCdV1j3idb8Y+GYN2/4O+F2zBthK9e3R5SvLd08bQGpuMZeP6Al4zriYNiicVQcyqaxUfKo5ndgYU39rD3r6Fi9yzqCaPiSCm+Ji+PaFfc9dp+XrI4yPDXUtxpbUvtNjB+HrIzx27chzpwUCTBscTk5hKTuTc12MzJj2Q1VZuS+D4VHdCe/aCfCcRPPUjWMZExPicnTusATSTs0YGklIkD8/eWNHjWPtGGPq7unlB4g/dopv2CgP51gCaafCugTwwp0TyTxdwp0Lt5Bf3PTj4BjTUby0/ih///wgN8XFcM/F/d0Op9WwBNKOjY8N5dnbL+DAydP86v1Et8Mxpk3acSKXxz7azewRPfn99aOb7arutsgSSDs3fUgEt0/py9LEdKuFGNMA725NppOfD09/axx+7fysqvqyo9EBXDu2N6UVlfxn98naCxtjzimvqOSTXWnMHN6zya8ebw8sgXQA4/qEEBPamY+8JpU5klVIcVm7v3TGmHo7mFFwbtK3DYezyS4s5ZoxvV2OqnWyBNIBiAjXjO3NuoNZ5BSWsu34KWb/9Qu++3I8lZVtdnQXY5rcqv0ZzPrrF/zirZ2oKkt2ptG1kx8zhkbUvnEHZAmkg7h6TBTllcpb8Sf40evb6eTnw5qkLBauO+J2aMa0CpmnS3jg7Z10CfBl8c5UXt98nE8T07h8RM+vTbtgPCyBdBAjorozIKILT366j5P5xbxyz2Rmj+jJH5fuIzElz+3wjHFVZaXywNs7OV1czjv3T2XKgB48/H4i+cXlXDPWmq9qYgmkgxCRc+24D8wZyoTYUP54wxjCugTwwNs7vza3sjEdyXvbU/jiQCa/umo4w6O688zN4wjrEkBIkP+5YUvM19lpBR3I3Rf3p194EPPGeq6kDesSwM9mDeHB93axKyWvww7HYMxbW04wKLIr376wLwA9uwfy6t2TKSgpJ8DPfmfXxI5MB9I90J/rx8d8ZXDFK0ZF4e8rfLgj9TxbGtN+peQWsfloDvPG9v7KRYIjendnUv8wFyNr/SyBdHDBQf7MGBrJkoRUKuyMLNMBLXFOb792nPV11JclEMO1Y3tzMr+EzUdy3A7FmBa3eGcqY/uEfG2aBFM7SyCGWcN7EuScupieV8z8hZt5Ysket8MyplnsSc3n8qe/4P3tyRzMKGB3aj7X2plWDWKd6IbOAb7MHtGTJQmpLN+TTlZBKauTMvnGhBhG9O7udnjGNKnff7KXAycL+NmbOxkQ3gURz3VSpv6sBmIAmDeuN6eLywkJCuDd+6fQPdCfp5btczssY5rUmqRM1h7M4n+uHMb3LhnA4axCpgzoQc/ugW6H1iZZDcQAcOnQSF68ayIT+4XRpZMf358xkD98uo8Nh7KZMrCH2+EZ02iVlcofl+4jOqQz86f2o5OfL3NG9aKXJY8Gc6UGIiJhIrJcRJKcv9VOICwi850ySSIy31kXJCIfi8g+EdktIk+2bPTtk4gwY2gkXZwRR+dP7UdUcCBPLt1n42WZduHjXWkkpuTz89lD6OTnGZpkQmwovUM6uxxZ2+VWE9aDwApVHQyscJa/QkTCgEeBycAk4FGvRPNnVR0GjAcuEpErWibsjiPQ35efzx7CzhO5vLj+qNvhGNMoWQUl/OajPQyP6s51NiVtk3ErgcwDXnLuvwRcV02ZOcByVc1R1VPAcmCuqp5R1ZUAqloKbANiWiDmDufGC2KYNTySJz/dx960fLfDMaZBVJVfvpNAfnEZf71pLL4+NqNgU3ErgfRU1TTnfjrQs5oy0cAJr+VkZ905IhICXIOnFlMtEblXROJFJD4zM7NxUXcwIsIfbxhDcJA/P1603eYPMW3SyxuO8fm+DB66YhjDo+yswqbUbAlERD4TkcRqbvO8y6lnFL96N7KLiB+wCPi7qh6uqZyqLlDVOFWNi4iwMf3rq0fXTvz1prEkZRSwYHWNh9mYVimnsJTff7KXGUMjuHNqP7fDaXea7SwsVZ1V02MiclJEolQ1TUSigIxqiqUAM7yWY4BVXssLgCRVfaYJwjXncfHgCKYNCuedrcn86LJBXxkvyJjWbElCKiXllfz33GH2uW0GbjVhLQbmO/fnAx9WU2YZcLmIhDqd55c76xCRJ4Bg4KctEKsBrh8fzfGcM2w7fsrtUIyps/e2pTCsVzdrumombiWQJ4HZIpIEzHKWEZE4EXkOQFVzgN8CW5zb46qaIyIxwMPACGCbiOwQkXvc2ImOZM6oXgT6+/DethS3QzGmTg5nFrDjRC7fmGBnXTUXVy4kVNVsYGY16+OBe7yWFwILq5RJBqwu2sK6dvJjzsheLElI45FrRpw7j96Y1uqD7Sn4CMwbZwmkudhQJqbOrh8fTV5RGSv32dlspnVTVd7fkcJFg8JtmJJmZAnE1Nm0QeGEd+3E+9uT3Q7FmPOKP3aKEzlFXG8XDTYrSyCmzvx8fbhmbBQr92dSUFLudjjG1GjJzlQC/X2YM7KX26G0a5ZATL1cOTqK0vJKVu6r7sxrY9xXWaks232S6UMizo3tZpqHJRBTLxNiQwnv2omlu9PdDsWYau1MziU9v5i5o6z20dwsgZh68fURLh/Zk5X7MmxoE9MqLU1Mx99XuGxYdSMkmaZkCcTU2xWjenGmtII1SVluh2LMV6gqS3enM3VgOMGd/d0Op92zBGLq7cIBPege6MfSRGvGMq3LvvTTHMs+Y81XLcQSiKk3f18fZo3oyWd7T1JWUel2OMac82liOiIwe4Q1X7UESyCmQa4YFUVeURnffm4TH+1MpbTcEolxz8GM0/zmo90sXHuEif3CCO/aye2QOgRLIKZBZg6L5OErh5OSW8SPFm3ngbd3uh2S6aC2Hz/FnGfW8OrGY8wYGsGT3xjtdkgdhiUQ0yA+PsJ3LxnA6v+6lDum9OXjXWlk5Be7HZbpgF7ZcIwgf1/WPXgZ/7x1AgMiurodUodhCcQ0io+PcNdF/amoVN7eakOcmJaVd6aMj3elMW98byK72ZhXLc0SiGm0/uFdmNw/jLfiT1BZWe/JJY1psA93plBSXsnNE2PdDqVDsgRimsTNk/pwLPsMGw9nux2K6SBUlUWbTzCyd3dGRQe7HU6HZAnENIkrRkXRPdCPRVtOuB2K6SASkvPYm5bPzZOs9uEWG2nMNIlAf1+uHx/Nq5uOU1BcxsWDI7hlUiydA2ziKdO0Nh/JYfHOFFbszSDQ34d543q7HVKHZQnENJmfzR6CiLD6QCYr9+/hWHYhv5k3yu2wTDtyNKuQmxdsINDfl6kDe3Db5L50D7QhS9xiCcQ0mZCgAB67diQAP3ljO+9tS+HBK4ZbLcQ0mTe2nEBE+PwXM+gVbGdduc21PhARCROR5SKS5PwNraHcfKdMkojMr+bxxSKS2PwRm/q4dVIsp0vK+Sgh1e1QTDtRWl7JO1tPcNmwSEserYSbnegPAitUdTCwwln+ChEJAx4FJgOTgEe9E42IfAMoaJlwTX1M6h/GoMiuLNp83O1QTDuxfM9JsgpKuXWydZq3Fm4mkHnAS879l4DrqikzB1iuqjmqegpYDswFEJGuwM+BJ1ogVlNPIsItk2LZfjyXvWn5bodj2oHXNx8jOqQzlwyOcDsU43AzgfRU1TTnfjpQ3fCZ0YD3eaHJzjqA3wJ/Ac6c70VE5F4RiReR+MzMzEaGbOrjhgnRBPj58Pomq4WYxjmaVci6g9ncPLEPvj7idjjG0awJREQ+E5HEam7zvMupqgJ1voRZRMYBA1X1/drKquoCVY1T1biICPvl0pJCggK4Zkxv3txygs1HctwOx7RRpeWVPPTeLgJ8fbhpYh+3wzFemjWBqOosVR1Vze1D4KSIRAE4fzOqeYoUwPsTE+OsmwLEichRYC0wRERWNee+mIb59dXDiQntzL2vxHMkq9DtcEwbo6o8/P4uNhzO5o83jqZnd+s8b03cbMJaDJw9q2o+8GE1ZZYBl4tIqNN5fjmwTFX/V1V7q2o/YBpwQFVntEDMpp5CggJ44a6JCPCdF7eQX1zmdkimDVmw+jBvb03mxzMHc/34GLfDMVW4mUCeBGaLSBIwy1lGROJE5DkAVc3B09exxbk97qwzbUjfHl34920XcCSrkPe3pbgdjmkjissq+MfnB5k1PJKfzRrsdjimGq5dSKiq2cDMatbHA/d4LS8EFp7neY4CdrlzKzdlYA+G9OzKxwlpzJ/az+1wTBuw+kAmBSXl3DGlHyLWcd4a2WCKpsVcPaY3W47lkJ5nE0+Z2i1JSCM0yJ+pA3u4HYqpgSUQ02KuGhOFKnyyK632wqZDKyqt4LO9J5k7Kgo/X/uaaq3snTEtZmBEV4ZHdWeJDW9iarFqfwZnSiu4ZkyU26GY87AEYlrU1WOi2HY8l5TcIrdDMa3YkoQ0wrsGMKl/mNuhmPOwBGJa1NXOL8pPrRnL1OBMaTkr9p3kCmu+avXs3TEtqm+PLozs3Z2lieluh2JaqdUHsiguq+TK0dZ81dpZAjEtbuawSLYdP0XumVK3QzGt0Mp9GXQL9COuX7UzPJhWxBKIaXEzhkVSqbA6KcvtUEwro6qs3J/BJYMj8Lfmq1bP3iHT4sbGhBDWJYBV+6ob/sx0ZHvS8sk4XcKlwyLdDsXUgSUQ0+J8fYTpQyJYdSCTiso6D8JsOoCVzo+K6UNs5Oy2wBKIccWMoRHkFJaSkJzrdiimFVm5P5MxMcFEdOvkdiimDiyBGFdMHxKBj3i+MIwBOFVYyvbjp5gx1Jqv2gpLIMYVIUEBTIgNPddkYczqpEwqFS6z/o82wxKIcc2lwyLZlZLH1mM2Qn9HV1GpvLrxGOFdAxgTHex2OKaOLIEY19wxpS99wjrz0zd3cNommurQnv3iEFuOnuKhK4bjY3OetxmWQIxrugX688y3xpFyqohHF+92Oxzjkp0ncnl6+QGuHhPFNyZEux2OqQdLIMZVF/QN40eXDea9bSl8tuek2+GYFlZZqfzsrR1EduvE764bbRNHtTGWQIzrfnTZILoH+vH5futQ72iOZhdyOLOQH80cTHCQv9vhmHqqcwIRkaCmelERCROR5SKS5PytdtAbEZnvlEkSkfle6wNEZIGIHBCRfSJyQ1PFZlqen68Po6KD2Z2S53YopoUlpuYDntEJTNtTawIRkakisgfY5yyPFZF/N/J1HwRWqOpgYIWzXPV1w4BHgcnAJOBRr0TzMJChqkOAEcAXjYBN6I0AAB+3SURBVIzHuGxUdDB7009TVlHpdiimBe1OySPA14fBPbu6HYppgLrUQJ4G5gDZAKq6E7ikka87D3jJuf8ScF01ZeYAy1U1R1VPAcuBuc5j3wH+4MRTqao2Kl8bN7J3d0rLKzmYUeB2KKYFJabmMSyqmw2c2EbV6V1T1RNVVlU08nV7qurZGYXSgZ7VlIkGvF83GYgWkbN13d+KyDYReVtEqtseABG5V0TiRSQ+M9Ouem6tRjnn/u+yZqwOQ1VJTMlnZG+77qOtqksCOSEiUwEVEX8ReQDYW9tGIvKZiCRWc5vnXU5VFajPiHp+QAywXlUnABuAP9dUWFUXqGqcqsZFRNgAba1V/x5d6BLga/0gHUjyqSLyisoYFd3d7VBMA/nVocx9wN/w1AhSgP8AP6htI1WdVdNjInJSRKJUNU1EooDqTr9JAWZ4LccAq/A0pZ0B3nPWvw3cXetemFbNx0cY0bv7uU5V0/7tTvX8WBhlNZA2q9YaiKpmqeptqtpTVSNV9duqmt3I110MnD2raj7wYTVllgGXi0io03l+ObDMqbF8xJfJZSawp5HxmFZgZO9g9qTm2xDvHURiSj6+PsLQXt3cDsU0UK01EBF5gWqamFT1O4143SeBt0TkbuAYcJPzWnHAfap6j6rmiMhvgS3ONo+r6tlBk/4beEVEngEygbsaEYtpJUZFB/Pi+qMcySpgUKR9qbR3ial5DI7sSqC/r9uhmAaqSxPWEq/7gcD1QGpjXtSpwcysZn08cI/X8kJgYTXljtH4M8FMK3O2LTwxJZ/YsC5sOZrD1IE97OrkduRQZgF+PkJsWBCJKXk2dHsbV2sCUdV3vZdFZBGwttkiMh3WoIiudPLzYfmek7yw7gg7k/N47o44Zo2o8SQ708bc/eIWUvOK+dGlg8gqKGVUb+tAb8sacvL1YMB+Npgm5+frw7Co7ny8K43DWYUE+Pmw/lBju9tMa5GaW8TR7DN0D/TnL8sPAF+evm3aprr0gZzG0wcizt90PH0QxjS5GyZEE9LZnyeuG8Uv30lg0xFLIO3F2ffyxbsm8sWBTNYkZVoCaePq0oRlvZmmxdwxpR93TOkHwKT+Yfz98yTyisoI7mwD7bV1mw7n0C3Qj+FR3RkVHcwPLh3kdkimkWpMICIy4Xwbquq2pg/HmC9NHhCGroD4oznMHG79IG3dpiM5TOoXhq9NGNVunK8G8pfzPKbAZU0cizFfMSE2lABfHzYfsQTS1mXkF3Mkq5BbJvVxOxTThGpMIKp6aUsGYkxVgf6+jO0TzMYjNmd6W7fJeQ8n9+/hciSmKdXlOhBEZBSeYdMDz65T1ZebKyhjzprUP4xnvzhMQUk5XTvV6eNqWqFNR7LpEuDLSDttt12py3wgjwL/cG6XAk8B1zZzXMYAnl+sFZXK1mOn3A7FNMKmwzlc0C8MPxu2vV2py7t5I56rxtNV9S5gLGDn3pkWcUHfUHx9hI2H7XTetiqroISkjAIm9w9zOxTTxOqSQIpVtRIoF5HueEbOtZ4w0yK6dPJj2qBwXtt4jMzTJW6HYxrgHyuSEIGZw+364/amxgQiIv8SkWnAZmcSp/8DtgLb8MzBYUyLeOSaERSXVfL4Eht0ua3ZdvwUL288xvwp/RjWy/o/2pvz9UoeAP4E9AYKgUXAbKC7qia0QGzGADAwois/vGwQf11+gG+Mj+bSYfZLti0oq6jkoXd30at7IA/MGep2OKYZ1FgDUdW/qeoUPKPeZuMZFXcpcL2IDG6h+IwB4L7pAxkc2ZVffZBIVoE1ZbUFf/ssif0nT/PbeaPsDLp2qi4TSh1T1T+q6njgFuA6YF+zR2aMlwA/H/78zbFkF5Ywf+Fm8ovL3A7JnMfLG47yz5UHuSkuxkZTbsfqchqvn4hcIyKvAZ8C+4FvNHtkxlQxtk8Iz377Ag6cPM09L8ZTVFrhdkimGh9sT+GRD3cza3hPfn/9aLfDMc3ofJ3os0VkIZAMfBf4GBioqjeranVT0BrT7GYMjeTpb41jy7Ec/rkyye1wTBUn84v55TsJXDggjH/eOt6u+2jnzvfuPgSsB4ar6rWq+rqqFrZQXMbU6OoxvZk7shevbjxOYUm52+EYLy+uP0p5ZSVP3TDWpqrtAM7XiX6Zqj6nqnYJsGl17rl4AHlFZbwdf8LtUIyjoKSc1zYeY+6oXsT2CHI7HNMCXKtfikiYiCwXkSTnb2gN5eY7ZZJEZL7X+ltEZJeIJIjIUhEJb7nojdsu6BvKBX1DeX7dESoq1e1wDPDWlhPkF5fz3YsHuB2KaSFuNlA+CKxQ1cHACmf5K0QkDHgUmAxMAh4VkVAR8QP+BlyqqmOABOCHLRa5aRW+e3F/TuQUsWx3utuhdHjlFZUsXHeEuL6hjI+t9regaYfcTCDzgJec+y/hOT24qjnAclXNcZrSlgNz8UyvK0AXERGgO5Da/CGb1mT2iF707RHEgtWHUbVaiJuW7k4n+VQR373Eah8diZsJpKeqpjn304HqThaPBrwbuZOBaFUtA+4HduFJHCOA56t7ERG5V0TiRSQ+MzOzyYI37vP1Ee6e1p8dJ3JttF4XqSr/t/ow/XoEMcsm/upQmjWBiMhnIpJYzW2edzn1/Hys809IEfHHk0DG4xlqJQHPWWNfo6oLVDVOVeMiIiIavjOmVbrxghhCgvxZsPqw26F0WFuOnmJnch53XzzApqvtYJp1fAFVnVXTYyJyUkSiVDVNRKLwjPJbVQoww2s5BlgFjHOe/5DzXG9RTR+Kaf+CAvy4/cK+/HPlQY5kFdI/vIvbIXU4/7fmMKFB/tw4IcbtUEwLc7MJazFw9qyq+UB1FycuAy53Os5DgcuddSnACBE5W6WYDext5nhNK3X7lL74+/jw/FqrhbS0Q5kFfLb3JLdf2JfOAXbdR0fjZgJ5EpgtIknALGcZEYkTkecAVDUH+C2wxbk97nSopwK/AVaLSAKeGsnvXdgH0wpEdgvk+vHRvB2fTE5hqdvhdCgL1x7B39eH26f0czsU4wLXhshU1Ww8Mx1WXR8P3OO1vBDPSMBVyz0LPNucMZq24+ZJfXgz/gQbDmVz1Zgot8PpMD7fl8HsET2J6NbJ7VCMC2ygGtMujOjdnQBfHxJSct0OpcPIOF1MWl4x4/uEuB2KcYklENMudPLzZVhUN3Yl57kdSoeRmOI51mNiLIF0VJZATLsxKjqYXSl5dlFhC9mVnI8IjOxtU9V2VJZATLsxJjqY08XlHMs+43YoHcKulFwGRnSli8022GFZAjHtxqjoYAASUqwZqyUkJOcx2jnmpmOyBGLajSE9uxHg53Oubd40n5P5xWScLrEE0sFZAjHtRoCfD8OjupOQbGdiNbezJyuMibEE0pFZAjHtyujo7iSm5FNpc4Q0q4SUPHzEc/q06bgsgZh2ZUx0CAUl5RzNttmXm1NiSh6DIrsSFGAd6B2ZJRDTrox2mlR2WT9Is1FVpwPdrv/o6CyBmHZlcGRXOvn5sP249YM0l9S8YrIKShgdbc1XHZ0lENOu+Pn6cPHgCD7YkUJBSbnb4bRLr248hghcMsTm1+noLIGYdueHlw0i90wZr2485nYo7U7umVJeXn+Uq0ZHMSCiq9vhGJdZAjHtzrg+IVw8OJzn1hymqLTC7XDalYXrjlJYWsEPLxvkdiimFbAEYtqlH88cTFZBKa9vPu52KO1GfnEZL6w7wpyRPRnWy/o/jCUQ005N7BfGhQPC+H9fHKK8otLtcNqF1zYe53RxOT+6bLDboZhWwhKIabfmT+lHxukSttkZWU3i08Q0JsSGnBtzzBhLIKbdmjY4HD8fYeX+DLdDafMyT5eQkJzHzOE93Q7FtCKuJBARCROR5SKS5PwNraHcUhHJFZElVdb3F5FNInJQRN4UkYCWidy0Jd0C/ZnYL4yV+yyBNNYqJwnPGGqn7povuVUDeRBYoaqDgRXOcnX+BNxezfo/Ak+r6iDgFHB3s0Rp2rxLh0WwL/00qblFbofSpq3an0lkt06MiLLOc/MltxLIPOAl5/5LwHXVFVLVFcBp73UiIsBlwDu1bW/MpUMjAc8XIMCTn+7jwXcT3AypTYg/msO1/1xLSm4RZRWVrE7K5NKhkXj+/YzxcCuB9FTVNOd+OlCfhtUeQK6qnr3MOBmIrqmwiNwrIvEiEp+ZmdmwaE2bNSiyKzGhnVm5P4NPdqXx7BeHeDP+BFkFJW6H1qq9vvk4Ccl5/PSN7Ww5ksPp4nIuHRbpdlimlWm2BCIin4lIYjW3ed7l1DOBdbONva2qC1Q1TlXjIiKs/bajEREuHRrJ2qQsHnw3gT5hnVGFz/acdDu0VqusopIVezPoE9aZLUdP8Yu3d+LvK1w0qIfboZlWptkSiKrOUtVR1dw+BE6KSBSA87c+vZzZQIiInB1HOgZIadroTXty6bAIisoqqFR49e7J9AnrzNLd6W6H1WptOpxDXlEZv7pqBNePjyYtr5iJ/cLoFujvdmimlXGrCWsxMN+5Px/4sK4bOjWWlcCNDdnedDxTBoQzITaEp24cQ98eXZgzohfrD2aTX1zmdmit0rLd6QT6+3DJ4AgenzeSCweEccukWLfDMq2QWwnkSWC2iCQBs5xlRCRORJ47W0hE1gBvAzNFJFlE5jgP/TfwcxE5iKdP5PkWjd60KZ0DfHnv+xdx5egoAOaO6kVpRaWd3luNykpl2e50ZgyJpHOAL90C/Xnj3ilcM7a326GZVsiV6cRUNRuYWc36eOAer+WLa9j+MDCp2QI07dqE2FDCu3biP7tPMm9cjedfdEg7knPJOF3CnFF2waCpnV2JbjocHx/h8pE9Wbk/g+IyG63X27LEdPx8hMuGWQIxtbMEYjqkOSN7caa0grVJWW6H0mqoepqvpgzsQXBn6zA3tbMEYjqkKQN60C3Qj2V2NtY5B04WcDT7DHNH9XI7FNNGWAIxHVKAnw8zh0Xy2d6TNty7Y2liOiIwe4Q1X5m6sQRiOqy5o3px6kwZm4/muB1Kq7BsdzoXxIYS2S3Q7VBMG2EJxHRYlwyJoJOfD8sSrRnrRM4Z9qTlM2ekNV+ZurMEYjqsoAA/pg+JYNnuk1RWNttoOm3C2b4gSyCmPiyBmA5tzshepOcXk5CS53YorlqamM7wqO7E9ghyOxTThlgCMR3azOGR+PkInyam1V64ncrIL2br8VPMtdqHqSdLIKZDCwkKYMbQSN7dmkxJece8qHDR5hOowrXjbLgSUz+WQEyHN39qX7IKSvlkV8erhZRVVPLapmNMHxJB//Aubodj2hhLIKbDmzYonAERXXhx/TG3Q2lxSxPTyThdwp1T+7kdimmDLIGYDk9EmD+lHztP5LLjRK7b4bSolzccJTYsiOlDbLI1U3+WQIwBvjEhmi4Bvry8/qjbobSY3al5bDl6ijum9MXHx+Y6N/VnCcQYoFugPzdeEMNHCansSc13O5xmp6r8edl+Ovv78s0L+rgdjmmjLIEY4/jxzMGEBgXwo0XbOFNa7nY4zerF9UdZuT+TX84dSnCQjbxrGsYSiDGOHl078fS3xnE4q5DHP9rjdjjNJjEljz98so9ZwyOt89w0iiszEhrTWl00KJz7pw/k36sOkXyqiLh+ocwbF93mT3Etr6jk/e0pbDqSw6r9GYR28eepG8ciYn0fpuEsgRhTxc9mD6FClS/2Z/K3FUm8vuk4a//7MgL82m6FfcGawzy1dD9hXQKYEBvKT2YOJqxLgNthmTbOlf8IEQkTkeUikuT8Da2h3FIRyRWRJVXWvyYi+0UkUUQWiog14pom4+/rw0NXDGfpTy9h4fyJZJwuadMXGZZVVPLy+mNMGxTO1l/N4rn5cYyOCXY7LNMOuPWT6kFghaoOBlY4y9X5E3B7NetfA4YBo4HOwD3NEaQx04dEMDCiCwvXHUG1bY7Y+2liOun5xdw9rb81WZkm5VYCmQe85Nx/CbiuukKqugI4Xc36T9QBbAZimitQ07H5+Ah3XtSfhOQ8th475XY4DbJw7REGhHexiwVNk3MrgfRU1bNtAulAg+bQdJqubgeWnqfMvSISLyLxmZmZDXkZ08HdMCGa7oF+LFx3xO1Q6m3b8VPsOJHLnRf1s4sFTZNrtgQiIp85fRRVb/O8yzm1iIa2DfwbWK2qa2oqoKoLVDVOVeMiIuwXmKm/oAA/bpkcy9LEdJJPnXE7nHpZuPYI3QL9uGGCVdJN02u2BKKqs1R1VDW3D4GTIhIF4PzNqO/zi8ijQATw86aN3Jivu2NKP0SEVza0nQEXU3OL+DQxnVsmxdKlk51waZqeW01Yi4H5zv35wIf12VhE7gHmALeoamUTx2bM10SHdGbuqF4s2nycwpK2cZX6yxuOoarcMaWv26GYdsqtBPIkMFtEkoBZzjIiEiciz50tJCJrgLeBmSKSLCJznIeexdNvskFEdojIIy0bvumIvnNRf/KLy3lvW7LbodTqTGk5izYfZ+6oXsSE2jS1pnm4Uq9V1WxgZjXr4/E6JVdVL65he6uPmxY3ITaEsX1CeGHdUW6b3LpHsH1/ewp5RWV856L+bodi2rG2e2mtMS1MRPjORf04nFXI5/vq3W3XYsoqKnlh3VFGRwdzQd9qr9E1pklYAjGmHq4YFUVMaGfuf20rf/hkL6eLy9wO6StW7s9g7jOrOZhRwPemD7ALB02zsgRiTD0E+Pnw3vencv34aBasOcwVf1tDUWmF22EB8OwXh7jrhS1UKjw/P46rx/R2OyTTzlkCMaaeIrsF8tSNY1l450SSTxXxVvwJt0OisKSc/111iOlDIlj200uYObxB1+YaUy+WQIxpoEuHRnJB31D+b81hyisqUVUeW7ybxxbvbvbXTkzJ45p/rOVIViEAizYfJ6+ojJ/MGtymRw02bYt90oxphPumDyT5VBEf70rjlY3HeHH9Ud7YcpyS8uZt1vpoZyq7UvK475Wt5BWV8fzaI0zqH8aEWOs0Ny3HEogxjTBzWCSDI7vy1NL9/HbJHqJDOlNcVsmO47nnyvxr5UHWH8xq8GuUlFfwyIeJnMj5chiV9YeyiQoO5EDGaa771zrS8oq5f/rARu2LMfVlCcSYRvDxEe69ZAApuUX0DunMG/deiI/AukPZAKTkFvGnZft5bm3DB2LcevQUL284xgvrjgKQd6aMxNQ8bp4YywOXD+VIViHDenVjxlAb6820LLsgz5hGmjcumhM5Z7h2XG/6hAUxOjqYDYeyYPYQPknwDDq9/fgpVLVBp9VuO+4ZRv6TXWn86qrhbDicjSpMHdSDC5wmq4sGhdspu6bFWQ3EmEYK8PPh55cPZVBkNwCmDgpn+/FcCkvKWZKQCsCpM2UczW7YSL7bneaw9Pxi4o+dYsOhLIICfBkbE4KPj/CDSwcxrk9I0+yMMfVgCcSYJjZ1YA/KK5X3tiWzMzmPb0yIBjy1kPpSVbafyOXK0b3o5OfDkoRU1h/KZmK/MDvbyrjOPoHGNLG4vmH4+wp//s8BAH46cwhdO/mda4oqLCnnp29sZ+uxnK9tW1peyUPv7WLF3pMAHMs+Q05hKdMGRXDZsEg+2J5CUkYBUwf2aLkdMqYGlkCMaWKdA3wZHxtKXlEZ4/qEENsjiHF9Qs41RX2wI4UPdqRy78tbSc0t+sq2T3y8h0Wbj/O7T/Y6tQ9P0pnQN4Srx/Qmv9gzlPzUgeEtu1PGVMMSiDHN4GwN4eoxUQCMjw1hX/ppzpSW8/qm48SGBVFSXsl9r26luMxzzchb8Sd4ecMxRvbuzuHMQjYezmHbsVy6BPgyOLIblw2LJCjAl+6Bfozo3d21fTPmLDsLy5hmcO3Y3mw4lM28cZ7+j/GxIVRUKq9uPMbu1Hx+O28kvYI7892X45nzzGqCO/uzL+00Fw3qwf+7PY6pf1jB65uPcySrgLF9QvD1EToH+HLf9IFUquLbioeSNx2HJRBjmsGAiK68+b0p55bH9/Gcbvv08iQ6+/syb3w03QP9+eMNo1mamA7A1WOj+NVVI+jayY9vTIjhtU3HqFS+coHgj2cObtkdMeY8LIEY0wJCuwTQP7wLR7IK+VZcH7oH+gPwrYmxfGti7NfK3zY5lhfXHwU8tRdjWiPrAzGmhZxNBLdM/nrCqGpwz25M7BfqbGfjW5nWyZUaiIiEAW8C/YCjwE2q+rWT5EVkKXAhsFZVr67m8b8D31HVrs0asDFN4K6p/enfowtjY4LrVP5/rvRcdR7WJaCZIzOmYdyqgTwIrFDVwcAKZ7k6fwJur+4BEYkD7KeZaTNGxwTzo5mD6zzkyPjYUL4/Y1AzR2VMw7mVQOYBLzn3XwKuq66Qqq4ATlddLyK+eJLLL5srQGOMMefnVgLpqappzv10oL7Tp/0QWOz1HMYYY1pYs/WBiMhnQK9qHnrYe0FVVUS0Hs/bG/gmMKOO5e8F7gWIja2989IYY0zdNFsCUdVZNT0mIidFJEpV00QkCsiox1OPBwYBB5225CAROaiq1TYWq+oCYAFAXFxcnROVMcaY83OrCWsxMN+5Px/4sK4bqurHqtpLVfupaj/gTE3JwxhjTPNxK4E8CcwWkSRglrOMiMSJyHNnC4nIGuBtYKaIJIvIHFeiNcYY8zWuXAeiqtnAzGrWxwP3eC1fXIfnsmtAjDHGBXYlujHGmAYR1Y7TrywimcCxBm4eDmQ1YTgtzeJ3l8XvLou/cfqqakTVlR0qgTSGiMSrapzbcTSUxe8ui99dFn/zsCYsY4wxDWIJxBhjTINYAqm7BW4H0EgWv7ssfndZ/M3A+kCMMcY0iNVAjDHGNIglEGOMMQ1iCaQORGSuiOwXkYMiUtPkV62CiPQRkZUiskdEdovIT5z1YSKyXESSnL+tejIuEfEVke0issRZ7i8im5z34E0RabXT9IlIiIi8IyL7RGSviExpS8dfRH7mfHYSRWSRiAS25uMvIgtFJENEEr3WVXu8xePvzn4kiMgE9yI/F2t18f/J+fwkiMj7IhLi9dhDTvz73R7eyRJILZzJq/4FXAGMAG4RkRHuRnVe5cAvVHUEnumAf+DEW9dZIFuLnwB7vZb/CDztDJx5Crjblajq5m/AUlUdBozFsx9t4viLSDTwYyBOVUcBvsDNtO7j/yIwt8q6mo73FcBg53Yv8L8tFOP5vMjX418OjFLVMcAB4CEA53/5ZmCks82/ne8oV1gCqd0k4KCqHlbVUuANPDMqtkqqmqaq25z7p/F8eUVTx1kgWwMRiQGuAp5zlgW4DHjHKdJq4xeRYOAS4HkAVS1V1Vza0PHHM0ZeZxHxA4KANFrx8VfV1UBOldU1He95wMvqsREIcaaUcE118avqf1S13FncCMQ49+cBb6hqiaoeAQ7i+Y5yhSWQ2kUDJ7yWk511rZ6I9MMzf8omGj8LZEt6Bs90xZXOcg8g1+sfqjW/B/2BTOAFpwnuORHpQhs5/qqaAvwZOI4nceQBW2k7x/+smo53W/x//g7wqXO/VcVvCaSdEpGuwLvAT1U13/sx9Zy73SrP3xaRq4EMVd3qdiwN5AdMAP5XVccDhVRprmrlxz8Uz6/c/kBvoAtfb15pU1rz8a6NiDyMp1n6NbdjqY4lkNqlAH28lmOcda2WiPjjSR6vqep7zuqTZ6vqDZgFsiVdBFwrIkfxNBdehqdPIcRpUoHW/R4kA8mquslZfgdPQmkrx38WcERVM1W1DHgPz3vSVo7/WTUd7zbz/ywidwJXA7fplxfstar4LYHUbgsw2DkLJQBPB9Zil2OqkdNf8DywV1X/6vVQg2eBbEmq+pCqxjizTd4MfK6qtwErgRudYq05/nTghIgMdVbNBPbQRo4/nqarC0UkyPksnY2/TRx/LzUd78XAHc7ZWBcCeV5NXa2GiMzF04x7raqe8XpoMXCziHQSkf54TgbY7EaMAKiq3Wq5AVfiORPiEPCw2/HUEus0PNX1BGCHc7sSTz/CCiAJ+AwIczvWOuzLDGCJc38Ann+Ug3hmqezkdnzniXscEO+8Bx8AoW3p+AO/AfYBicArQKfWfPyBRXj6a8rw1ADvrul4A4LnrMpDwC48Z5u1xvgP4unrOPs//KxX+Yed+PcDV7gZuw1lYowxpkGsCcsYY0yDWAIxxhjTIJZAjDHGNIglEGOMMQ1iCcQYY0yDWAIxppFE5GFn9NoEEdkhIpOb8bVWiUhccz2/MfXhV3sRY0xNRGQKnquFJ6hqiYiEA61mqHNjmpPVQIxpnCggS1VLAFQ1S1VTReQREdnizKmxwLmq+2wN4mkRiXfmCpkoIu8581Y84ZTp58wF8ZpT5h0RCar6wiJyuYhsEJFtIvK2M/6ZMS3GEogxjfMfoI+IHBCRf4vIdGf9P1V1onrm1OiMp5ZyVqmqxgHP4hli4wfAKOBOEenhlBkK/FtVhwP5wPe9X9Sp6fwKmKWqE/Bc+f7z5tlFY6pnCcSYRlDVAuACPJMTZQJvOoPgXerM4LcLz4CQI702OzuW2i5gt3rmcCkBDvPlQHknVHWdc/9VPEPUeLsQzwRn60RkB57xnvo26c4ZUwvrAzGmkVS1AlgFrHISxveAMXjGWTohIo8BgV6blDh/K73un10++z9ZdYyhqssCLFfVWxq9A8Y0kNVAjGkEERkqIoO9Vo3DM8gdQJbTL3Hj17esVazTQQ9wK7C2yuMbgYtEZJATRxcRGdKA1zGmwawGYkzjdAX+ISIheCb+OYinOSsXz2i26XimBKiv/Xjms1+IZzj1r8zdraqZTlPZIhHp5Kz+FZ5Ro41pETYarzGtjDMV8RKnA96YVsuasIwxxjSI1UCMMcY0iNVAjDHGNIglEGOMMQ1iCcQYY0yDWAIxxhjTIJZAjDHGNMj/B22bEVUzrx2PAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nOy9d5wcd33//3pvL7d7e10n3alasmS5YFs2NjYY3DAYMNWhBAgJIbSQQvmFEAIJIZhOiGkGvgEDMRhjSmyDK7Zxt2TZlmVJVtcVXW/b6+f3x2c+s5+Zndlyt3t70n2ej8c97m52duazszOfd39/iDEGhUKhUChkHM0egEKhUCiWHko4KBQKhaIEJRwUCoVCUYISDgqFQqEoQQkHhUKhUJSghINCoVAoSlDCQWELEd1PRO8t8/p3iejTizmm5Q4R/Z6I3l3H45X9jhXLFyUcThCI6AgRJYkoRkQjRPQjImpZxPP/BRE9JG9jjL2fMfa5BpxrLREx7bPGtM/+T9LrREQfJ6L92jU5RkT/SUQeaZ8face4xnTsr2vb/0L6XHnpXDEiut5mXPdr7z3LtP3X2vaX1/M6WMEYexVj7MfS2B+q9J5GUOk7Upz4KOFwYvFaxlgLgBcBOBvAJ5s8nkYT0T7v2wD8KxFdpW3/JoD3AXgXgBCAVwG4HMDPTe9/QdsHAEBELgDXAjho2u9RxliL9PPhMmMyH7MDwIUAxmv9cCcJ4jt6M4BPE9EVzR6Qoj4o4XACwhgbAXAnuJAAABDRBUT0CBHNENEzsharaZiHiChKRIeJ6B3a9s8S0U+l/YQ26JLPR0RbAHwXwIWaljijbf8REf2H9vfLiWiQiD5KRGNEdJyI3iMdo4OI/o+I5ojoSSL6j2q1XsbYowB2AzidiDYC+CCAdzDGHmWM5RhjuwG8CcDVRHSJ9Nb/A3AxEbVp/18F4FkAI9Wc14afAfgzInJq/78NwK8BZKTPej4RPap9F8eJ6HqTVXMlEe0jolki+jYRPSBcO8IaIKKvENG09n29Snrv/UT03jLficFNZLYuiOgKItqrnft6ACR/OCL6SyLao537TiJaU81FYYxtB/+O5HvS8lia5fd17T6ZI6JdRHS69tqPiLsr79bu1wfkMRDRS7T7Z1b7/RLTtfkcET2svfcuIurUXvMR0U+JaFL7Xp4koh7ttVYi+qH2XQ1p96b4fpctSjicgBBRH7i2fED7fxWA2wH8B4B2AB8D8Csi6iKiILim/SrGWAjASwA8Xcv5GGN7ALwfRQ07YrPrCgCtAFYB+CsA35Im5m8BiGv7vFv7qeazEhFdBGArgJ0ALgMwyBh7wjTGAQCPAbhS2pwC8FsAb9X+fxeAG6s5bxmGATwvncfqmHkA/wCgE9yquAxcoEGbrG4Bt/o6AOwD/05kXqxt7wTwJQA/JCLDJF7Dd6KjnftWAP+iHfsggIuk168B8M8A3gigC8CfANxU6bjaey8AcDqK92S5Y10J4GUANoHfL9cCmJQO9w4An9PG+DS4QAYRtYPf598Ev3ZfA3A7cetN8HYA7wHQDcAD/iwA/H5rBdCvvff9AJLaaz8CkANwCrhFfiWAZR+HUcLhxOI3RBQFMABgDMBntO1/DuAOxtgdjLECY+xuANsBvFp7vQCudfsZY8c1TbsRZAH8O2Msyxi7A0AMwKmaFvYmAJ9hjCUYY88D+HEVx5sAMAXgBwD+iTF2L/iEcdxm/+PgE5HMjQDeRUQRAJcA+I3F+y7QtEnxc0GFcYljbgZ3qzwqv8gY28EYe0yzao4A+J52boB/J7sZY7cyxnLgE53ZkjnKGPs+YywPfp16AfRUGFM1iHPfwhjLAviG6dzvB/AFxtgebWz/CeBFFayHCSJKAngUwLdRvL7ljpUFdwduBkDaPvJ3ejtj7EHGWBrAp8Cto34AVwPYzxj7iXZtbwKwF8Brpff+D2PsBcZYEsDNKFoyWXChcApjLK99R3Oa9fBqAH/PGIszxsYAfB1FhWLZooTDicXrNe3/5eAPVqe2fQ2At8gTHICLAfQyxuIA/gz8YT1ORLdrk1ojmNQmAkECQAv4hO0CF2oC+W87OhljbYyxLYyxb2rbJsAnSyt6tdd1GGMPaef/FIDbtEnDzGOMsYj081iFcd0K4FIAHwbwE/OLRLSJiG4jnjgwBz4xiu9qJaTPznjny0HTIUak1xPan/VIPrA6t/w9rAHwX9I9NAXudlpV5pid2tg+Cn5fuisdizF2H4Drwa3JMSK6gYjC0jHlMca0967Ufo6azn/UND5Z2In7D+Df050Afk5Ew0T0JSJya+N0gz8bYqzfA7c8ljVKOJyAMMYeADeFv6JtGgDwE9MEF2SMXaftfydj7ArwyXMvgO9r74sDCEiHXlHutAsY8ji42d4nbeuf57HuA9BPROfLGzXN8gIA91u856fgk9dCXUoA9An79wA+AAvhAOA74Nd5I2MsDO5eEW6h45Cug+Yu6is5QpVDsdhW7js9Dum6a+eWv4cBAH9juo/8jLFHyg6Ca+JfA3fjfbCaYzHGvskYOxfAaeDupY9Lh5TH2ALuKh3WfsxWzGoAQ+XGp50vyxj7N8bYaeBuvNeAuwQHAKTBFRExzjBjbGulY57sKOFw4vINAFcQT6v8KYDXEtEricipBd9eTkR9RNRDRNdosYc0uKunoB3jaQAvI6LVRNSK8tlPowD6SAqsVovmHrkVwGeJKKBZLu+q8Da7Y70AHoj9GfEgvJOItgL4FYBHANxj8bZvArgCwIPzOacN/wzgEs1tZCYEYA5ATPusH5Beux3AGUT0euKB/w+hvFAuh9V38jSAN2rX+RTw2I987q1E9Ebt3B8xnfu7AD6pXU8RqH1LDeO5DsAniMhX7lhEdB4RvVjT3OPgQqUgHefVRHSx9rk+B27ZDQC4A8AmIno7EbmI6M/AhcttlQZGRK8gojM0F+ccuJupoLmz7gLwVSIKE5GDiDaQMbFhWaKEwwkKY2wcXBP+V+3BEQHAcXBt6OPg368DwD+Ca11T4L7vD2jHuBvAL8AzeHag/EN2H3g2yggRTZTZz44PgwcER8C17ZvAhdV8+DB4HOKn4K6D58DdC69njBXMOzPGphhj92pulLrAGBvWXFZWfAw8MBoFt9J+Ib1vAsBbwAPNk+CT23bM71pYfSdfB8+cGgWPV/zM4tzXaefeCOBh6fVfA/giuOtlDvy66plSVXA7gGkAf13hWGHw6zIN/r1NAviydJz/BY+nTQE4FzymBsbYJLjG/1HtPZ8A8Brtc1ViBXgiwByAPQAeQNHqexd48Pp5bUy3wN51uWygOj4vCkXVENEXAaxgjC242peI/g3AGwC8jDE2s+DBLSJE5ACPObyDMfbHZo+n2RDRj8Cz0f6l2WNZ7ijLQbEoENFmIjpTS009H9zd8et6HJsx9hkAN4DHHJY8mvsvQkReFOMRlYLgCsWi4qq8i0JRF0LgrqSV4C6Pr4LXINQFxphly4slyoXgrhPhyni9TRaVQtE0lFtJoVAoFCUot5JCoVAoSmiaW0nLS78RvPKTAbiBMfZfWon8LwCsBXAEwLWMselyx+rs7GRr165t6HgVCoXiZGPHjh0TjDFzVwEATXQrEVEveAXvU0QUAk+lfD2AvwAwxRi7jngL4DbG2P9X7ljbtm1j27dvb/iYFQqF4mSCiHYwxrZZvdY0t5LW4+cp7e8oeO7xKvB8fdF358fgAkOhUCgUi8iSiDkQ0VrwboiPA+iRmnCNwKbhGBG9j4i2E9H28fHl2kpfoVAoGkPThYPWO+VX4F0R5+TXtIpWS78XY+wGxtg2xti2ri5Ll5lCoVAo5klThYPWW+VXAH7GGLtV2zyqxSNEXGKsWeNTKBSK5UrThIPWEfKHAPZoHR0Fv0NxIZh3o46FUgqFQqGojmZWSF8E4J0AdhGRWJnsn8Gbgt1MRH8F3pTr2iaNT6FQKJYtTRMOWkdLsnn5ssUci0KhUCiMND0g3UwOjMXwtbtfwEP7J6DaiCgUCkWRZd14b8/xOVx/334UGHD1mb34+rUvgse1rOWlQqFQAFjmwuG1Z63EFaf14IcPHcaX79yHVr8b//mGM5o9LIVCoWg6y1o4AIDP7cSHXnEK5lJZfO+BQ7j6jF5cdEpn5TcqFArFSYzyoWj8w+Wb0Nfmx5f+sFfFHxQKxbJHCQcNn9uJv3nZejwzOIudAyfUSpMKhUJRd5RwkHjjOX0I+Vz40cNHmj0UhUKhaCpKOEgEvS68+dw+/P6544imss0ejkKhUDQNJRxMvPqMXmTzDA+8oDq9KhSK5YsSDibOWd2GjqAHd+0ebfZQFAqFomko4WDC6SBctqUbf9w7hkyu0OzhKBQKRVNQwsGCSzf3IJrOYdeQylpSKBTLEyUcLNi2tg0A8OSR6SaPRKFQKJqDEg4WdLZ4sb4ziO1Hppo9FIVCoWgKSjjYsG1tG3YcnUahoKqlFQrF8kMJBxu2rWnHdCKLQxOxZg9FoVAoFh0lHGw4Z00EAPD0wGyTR6JQKBSLjxIONqztCMLjcmDfyFyzh6JQKBSLjhIONricDmzsbsHekWizh6JQKBSLjhIOZTh1RQj7lHBQKBTLECUcyrB5RQhj0TSm45lmD0WhUCgWFSUcynDqijAAKNeSQqFYdijhUIbNK0IAoILSCoVi2aGEQxm6Q16EvC4cnog3eygKhUKxqCjhUAYiQn97AMemEs0eikKhUCwqSjhUYLUSDgqFYhmihEMF+tv9GJxOqh5LCoViWaGEQwVWtweQzhUwHks3eygKhUKxaCxZ4UBEVxHRPiI6QET/1Kxx9LcHAAADyrWkUCiWEUtSOBCRE8C3ALwKwGkA3kZEpzVjLEI4qLiDQqFYTixJ4QDgfAAHGGOHGGMZAD8HcE0zBrIq4geREg4KhWJ5sVSFwyoAA9L/g9o2HSJ6HxFtJ6Lt4+PjDRuIz+3EirBPCQeFQrGsWKrCoSKMsRsYY9sYY9u6uroaeq5VET+Oz6Qaeg6FQqFYSixV4TAEoF/6v0/b1hR6Wn0YnVPCQaFQLB+WqnB4EsBGIlpHRB4AbwXwu2YNpiekhINCoVheLEnhwBjLAfgwgDsB7AFwM2Nsd7PGs6LVi3gmj2gq26whKBQKxaLiavYA7GCM3QHgjmaPAwB6wj4AwOhcCiGfu8mjUSgUisazJC2HpYYQDiOzqkpaoVAsD5RwqIKukBcAMBlXwkGhUCwPlHCogs4gFw4TMbVcqEKhWB4o4VAFYb8LLgdhUjXfUygUywQlHKqAiNDR4sGkshwUCsUyQQmHKukIelXMQaFQLBuUcKiSjhaPijkoFIplgxIOVdLZ4sWEijkoFIplghIOVRIJuDGbUBXSCoVieaCEQ5W0+t2IpnPI5QvNHopCoVA0HCUcqiTi520z5lK5Jo9EoVAoGo8SDlXSGuDCYSahgtIKheLkRwmHKon4PQCA2aSKOygUipMfJRyqJKy5lWaUcFAoFMsAJRyqJKK5lVTGkkKhWA4o4VAlrZrloNxKCoViOaCEQ5Uo4aBQKJYTSjhUidvpgMflQDytUlkVCsXJjxIONdDidSGmhINCoVgGKOFQA0GvU1kOCoViWaCEQw0EPS7E0vlmD0OhUCgajhIONdDidSGRUZaDQqE4+VHCoQaCXpdyKykUimWBEg41EPQ6VUBaoVAsC5RwqIGgx4W4ijkoFIplgBIONaDcSgqFYrmghEMNtHhdiGdyYIw1eygKhULRUJRwqIGg14UCA1JZtRqcQqE4uWmKcCCiLxPRXiJ6loh+TUQR6bVPEtEBItpHRK9sxvjsCHqdAIC4SmdVKBQnOc2yHO4GcDpj7EwALwD4JAAQ0WkA3gpgK4CrAHybiJxNGmMJHie/XJmcshwUCsXJTVOEA2PsLsaYUL8fA9Cn/X0NgJ8zxtKMscMADgA4vxljtMLjUsJBoVAsD5ZCzOEvAfxe+3sVgAHptUFtWwlE9D4i2k5E28fHxxs8RI4uHPJKOCgUipObhgkHIrqHiJ6z+LlG2udTAHIAflbr8RljNzDGtjHGtnV1ddVz6LYIt1JaBaQVipOWe54fxY2PHkEqu7xrmlyNOjBj7PJyrxPRXwB4DYDLWDE3dAhAv7Rbn7ZtSVC0HJb3TaNQnKzc9MQxfPLWXQCAB18Yx/fftQ1E1ORRNYdmZStdBeATAF7HGEtIL/0OwFuJyEtE6wBsBPBEM8ZohRAOaRVzUChOOpKZPL70h7148bp2fOzKTbhnzxgeOjDR7GE1jWbFHK4HEAJwNxE9TUTfBQDG2G4ANwN4HsAfAHyIMbZk1HSvCkgrFCct9+wZxXQii49cthHve9kGdAQ9+PmTA5XfeJLSMLdSORhjp5R57fMAPr+Iw6kaj5Nn1S4n4ZAvMDDG4HIuhdwFhaJx/OG5EXSHvLhgfQecDsLVZ/biF08OIJnJw+9ZMhn1i4Z64mtguWUrpbJ5vP37j+GCL9yLZwdnmj0chaJhMMbw6KFJXLyxE04HjzFcurkb6VwBO45ON3l0zUEJhxpYbnUOv316CI8fnsJELIOv3vVCs4dTNQ8fmMBf/ehJ7B6ebfZQFCcIB8ZimIpncMH6Dn3beWvb4XIQHjm4POMOSjjUwHKLOfxm5zDWdwbxkcs24oEXxjEZSzd7SBVJZvJ4/0934N69Y/jbm3aiUFBNEhWV2TXEFYmz+/VOPgh6Xdi6MoynB5an1ayEQw0sJ7dSRjOnLz+tBy/b2AkA2H4CmNcPvDCGaCqHa7f14dB4HI8emmz2kBQnAHuOz8HjcmBdZ9CwfeuqVjw3NFv3TsyMsSXf3VkJhxpYTm6lvSNzyOQLeFF/BGf0tcLjcmD7kamGnzeTK+Cvb9yOq77x4LzWznjghXGEfS58+jWnwe0kPPjC4lTPK05s9hyP4tSeUEnixekrWzGXymFgKlm3c82lsnjDtx/B6Z+5E/ftHa3bceuNEg41oFdILwPh8MwgN7PP7GuF1+XEpp4W7BuNNfy89+wZxd3Pj2LvSBQ3Pnq05vc/fzyKrStbEfK5cfbqNjy8TP3Fito4OB7Dxu6Wku2be0MAgBdGo3U713fuP4hnBmfgdBA+/stnl2wlthIONSB3ZU1l80veLFwIh8ZjCHicWBXxAwDWdbbg8ETjhcNtzw6jJ+zFGata8ce9YzW9t1Bg2D8axakr+AO9bU0b9h6PIp2r38M3NJPENd96GB/82Q7kloF7cTmQyuYxMpfC6o5AyWsbOrnAOFSnez+XL+AXTw7gytN68N13novJeAa3PXu8LseuN0o41IDDQXA7Cb96ahCbP/0H3LJjsNlDahjHJhNY3R7QWwes6wxiaDpZ14nWil1Ds9i2th3nrW3Hs0MzyNYwAQ9OJ5HI5LFZEw5besPIFRj219Hi+fztz+OZgRncsWsEv9i+fAukTiaGZpJgDFjdXiocWgNudAQ9ODQer8u5nh2axVQ8g9edtQoXru/AylYf7to9Updj1xslHGrE43RgcJr7H3cPzzV5NI3j6FQCayRNan1nEAUGDEwlyrxrYcwmsxiYSmLryjDOWRNBKlvAvpHqzfnBaT42oQFu6Q0D4MHGehBNZXHP82P4y4vW4dSeEG5fohqfojaOafe0lXAAgPVdwboJh0cP8gSJCzd0gIjw8s3deOjAxJKMYyrhUCMiKA0Ax2frF6RaShQKDMemEljTUczcWNXG3UvHZ1MNO+9ebRLf0hvGes2cP1aDMBqZ42NbEfYB4NaOx+nA/rH6WA4P7Z9AJl/AVaevwCs2d+OJw1OIzSNorlhaDFYQDqvbg7risVB2HpvGKd0taA96AAAv2dCBRCaPvSNLT9FUwqFGjMKBT0bxdO6kqqKcSmSQyRX0eAMAdLV4AQDj0WKtw23PDuOlX7oPX71rX13iL8IiW9sR1IVRLZbK6BwfW48mHJwOQn+7H0cn66P17RyYgcflwIv6Izh/XRtyBaYLtOXC0EwSv9w+cFLVj4zOpeF0EDq0e9zMqogPI3OpmlycduwdieoWLQCcvboNAJZkLYUSDjUiC4fhGS4c/u7nT+NN33mkbpNQs5nQit06pYelM2QUDtFUFp+8dRcGppL47/sO4I5dC/ebCkust9WHVr8bYZ9LFxhW/PbpIVz61fsxPMP3GZ1LIeR1Iegttgxb2xHE0cn6aH3PDMxgS28YHpcDm3p4XGNfHbNYljrpXB6v+++H8PFbnsUPHjrU7OHUjfFoGh1Bj942w8zKiB8Fxu+vhRBNZTE4ndRjYgCwstWHrpAXTx9TwuGExyPlQU/E0oilc7hnD89VvvWpJbP0xIKYjGUAAJ0tHn1b0OOE3+3UhcOjBycRTeXws/e+GFt6w/jiH/YuOHtneDaF9qAHPjdvctbfHsCAjTmfLzD83c+fxqHxOH76GE95HZ1LoTts1P7WdnLhsFDLhjGG3cNzOGMV1/pWRfwIepw1xUROdB4/NIXJOL83bt5+8iRjjMfS6ApZWw1A0aUqlMH5ItybQrEAACLClt7wklQylHCokVa/2/C/nP/83NDJ0ctHWA6ymU1E6Ap5Ma699sjBSfjcDmxb24a/v3wjjk0lcPuuhQVoj88k0dvq0//vbfXjuM0DeXC8GEf4035eyzAWTesuJcHq9gCS2bw+7vkyHuWKwMZu/mATEU7pCRnGcbJz394x+NwOfPSKTTgwFtMtthOd8Wh54bBSc68OzSzMAhUu0rWmlNlN3S04MBZDfom56pRwqBExYQqt+sgEdyX53U5bLfdEQ1gHXSYfbFfIq7+2e3gWZ66KwOty4ootPdjY3YJv//HggnzRx2dT6G0txjnag27MJDOW+z6rFem9ZEMHjkzGwRjDXDJbIryFsBmdXZhwOKx9z2ul9gqrIr6GBuiXGs8Pz+H0la245FS+LO/J0ql3PJouuddlRIKDiGnNF+EiFZaIYNOKENK5Qk3JF4uBEg41IiYf4Y8/ovmzz13ThsHp5AlZGHfbs8P4yaNH9Il9Mp6B20kI+43LfbQHPbrL6dhUQk8ZdTgIH3zFBuwbjeLeGgvXZCbjGXSFiq6sSMCD6UTW8pruPT4Hn9uBV5zajWgqh+lEFtFUDiGfccwrNOGw0MyyI1o8aZ2UwbUi7MfIbKou33kuX8CPHzmypBMb9o9FsbGnBeu7RGHYiR9jKxQYJiq4lYJeF/xuJyai1sIhly/gy3fuxX/ds7/svTA4nURH0IOAx3iPbtCu52IUmdaCEg414tf84SIVTVgO56xpQyKTx3Qi27SxzYedx6bx4f/diU//djdueYr7kSeiaXQEvSVr54a8LsTSOaSyeYzOpQ2pf689cyX62vy4/o8H5j1ZziWzCEuafyTg1qrRS2MZI3PcyljfxSfrI5NxxNI5tHiNlkNR61uYhn9kMgG3k7AyIru9fEhk8ojWIZ31+j8ewGd+txtv+s4jGFqC7prJWBrTiSxO6Q6hxetCd8iLw1Lufyqbx83bB3QL60QhlskhV2D682xHZ8iju1vN3Lx9EN/640F8/Z4XyrpWB6cT6DNZDQDQ3y4y85bW966EQ4343PySubXA9JHJODwuB7au5IFKkQ9dKLCGtVcoFBj+uG+sLrnXP3v8GEJeFzpbvPi1FlCfS5W6ZwCuQcUzOd08loWDy+nA+y/ZgGcGZvRCn1pIZfNI5woI+4rnbQvwB3Y6UepaGtP8xP3aGAamElw4mCyHjhYvXA7SayBkhmaSiKaqE+bDM0msaPUZGrMJq2Rkga6lrGY1bF4RgstBeoB9KSGshA2aMF7XGTQIgi/+YS8+ccuz+PD/PnVCWc9zSf79my1OM10tXkzErF2ct+wYwMbuFvSEvfi/Z4ZtjzE8k9TjF+Zj+9yOhhaYzgclHGpEWA5CqT46mUB3yKtrHjOJLEbnUrjwunvx7v95oiFj+Ma9+/Ge/3kSf/6Dx+fVuVSQyRVw1+4RXLG1B687ayV2DkyjUGCIp/MIekuXRWzxuRBL5fTYilkLevO5fegOefGt+w9Udf4DY1F84fd7kMsX9IfUYDlof08nMmCM4Z7nR5HI8M87Hk2jO+TV9xG+/7DpIXc6CN0hb0ls4N49o3jpF+/D277/WFUtQUZmU+gJGYPdvbrLih97NpHFJ255Bk8crq177WOHJjGdyOIfr9iEs1dH8FgVbcYZYzg0Hlu0egNheQmB2N8e0C2cbL6A3+zkisXu4Tk8eaT5rrHr79uPP//B4xUTBqIpfj+FfKXKkExni9dQ41N8fxY7B2Zw9Zm9uPK0FXjwhQnbwPJkPGNIDxcQEfra7DPzkpk87nl+FDMWSlIjUcKhRkTVsNCaZ5NZtHhduuYRTeVw754xjM6l8fCBybovkJPJFXDjo0cAcFfHbc/aayqVePjABOZSOVx9Ri/WdwWRyhZwfC6FWDpnqBUQtHhdyBUYRrXJ0Fw05HM78dcvXY+HD0xaBivNguyTt+7C9x44hJ8/OYA5TYNvNbiVuMCdTWTxy+2DeO+N2/H9Bw8DAMbmUugO+XRhIjJnWizG3dPqK3Erffv+gygw4LmhOdy7p3KcZCyaRk+rUTgIhWBaS+/8wu/34Obtg/jAT3fU1IPqycNTcBBw0Smd2La2Hc8NzSKZKf/+nzx2FJd+9QH8+23PV32ehSAmxm5NQHa08PgTF1JxTCey+PdrtoIITV857f59Y/jKXS/goQMT+PRvnjO8NhXP4M3feQT/8IunARSFQ7iScAh5Ld1Ku4ZmwRjwov4IzuxrRTKbt3StZfMFzCSy6Gixdl/1t/kt3UqFAsN7b3wS771xO9747UcWtSJfCYcaeeM5q/C9d56Lv7p4nb7N63Lomkc0lTWktz50oL4Pyt6ROcwksrj+7WdjfVcQv95ZubYiX2D4+C+fwVXfeBBPSmsy3L7rOEI+Fy7e2Kn77g+Px5HI5BD0lE6yQW2R9WFNOFi5nt56fj/cTirxvX7h93uw9TN34iEt7TSVzeOZAZ5xdPfzo5gVloOk+bcFheWQxa+0eMizgzOIpXOIZ/LoDnvhdTngdlJROFi4B3pbfQbXz8hsCjuOTuNjV25C2OfCfRWC6IwxjMym9PiFQHz+uRQPmv9p/wQcxDXEWoqadg7MYPOKMIJeF/JbofoAACAASURBVM7qiyCbZ9g/Zp/3zhjD9fdx6+wnjx3Vr91COKplfNkxFk3D7STdUusMepHJFxBN5/SxnrumDVtXhquyfBrJzx4/hq6QF3932UY8cnASY9Hid3/bs8PYfnQav945hGOTiardSp0tXkwlMiVWwS69tX2kbC8voUDYVWH3tQUs3cQPH5zAwwcm8YpTu3BoIo6bn1y8Zo9KONQIEeGVW1cYNGuPy6FPatFUDntH5nD6qjCIUHXDLsYYbnjwIL557/6y+4kUzrP6Injl1hXYfmS6omvplh0D+OWOQewdieIDP92B2US26FI6rQdel1PPmDg0EdPcShbCQdt2XJuIrR6okM+N89a24/69xUV2svkC/ufhIwCAX+7gN/fgdEJfUW9gKoG5JP8MBsvBr7nqkhndhbH96LSeNdLZwoPmYZ9bj4NYuQd6wkbhsP0oF5Av29SFi07prDiZzaVySGbzJcJBWC2zCV75OjSTxD9cvglEqHoFOsYYnhmYwVna8pRrO7lFWi6t8dBEHGPRNN5ybh/yBYb7980/QwzgLrZLvnw/vnLXPtt9xubS6GzxwqFVEQsNeDKWwYGxGIh41s1ZfRE8PzzXtLhDMpPH/fvG8LqzVuKyLd0AYIiB3bl7RF/u9969o4imqxMOEb8bjKEkRnVgLIYuza28sacFLgdZFkaKeEWnTeC7v92PuVSuRND/ascgIgE3vvvOc3H26ghuXsROwEo4zJOAp+iT97qcCHpcIOI3z+GJOLasCGNF2Gfb/uGh/RN6phMAPHlkGv95x1587e4XMFYms+a5oVm0Bdzoa/PjJRs6kCswPFFhhbZf7RjiXUQ/cjGm4hl88c69uHfPKOZSObz2zJUAeFDMQcVirxaLmIN4gIZnkwh4nHpQ3swF6zuwbzSqm8C7hmaRyRXgczt0y0FMfuevbcfgdFIPOssxBxH8T2byGJlNwekgzCazunkvxhP2u8u6lVaEfYhn8vqDvePoNPxuJ7b0hnFabxiD08my5vq4pnmaq6/dTgcCHifmUlndt33hhg6c2hPCM1X2ypmIZTCXymFTDxfO/W1cOJRr+SFW5PubSzYg6HFiZw1WynNDsyUB7+8+cBAA8P0/Hbb1l4/HeIxHIDTgyVgaB8fj6Gvzw+d2YlNPCHOpHMZs0j4bzfajU8jmGV66sRNbV7aixevCU1J68N7jUbzxnFVoC7hxYCxWdCtZWMEyQmkxT97HphJYo7mYvS4nesI+y2yzybim0NikzIrvXQ5KC2v00lO74XU58arTV2DvSHTRig8rCgciWkNEndrfFxDRx4joDY0f2tLG55KFgwMOB6HF68JcKoeZRBbtQQ/62vyWpuJELI0//+HjePlX7tfNTaHNAjDUCjw7OGMQIkcnE1jf1QIiwrY17XA6qOzyndPxDLYfncIrt/Zg68pWvOeidfjfx4/hwzftxKqIHy/bxAuaHA5C2O/GTCKLuE3MQWwbnkmV9dEK83qf1mlSmN5vPW81JuMZxNI5HNMmv4s3diKTL+jtA1oNwoFf44GpBHIFhjNWtfL/tWsqXF8hH7/u4m8zIogq4g57j0exuTcEt9OhLwy0v0z7gqk4nxCs0h3DPjdmk1k9KN0b8WNDd0vVNQBCqAjLLahljpXLXDk8kYDHydc73tgTqrqj52wyi9f890P4l988h0c0d2ehwNuCtHhdyOQKej2HGXMVcYd2LSZiGYxHiy43vedUk9qKPH5oCk4H4by1/NlY1xnEYe1em45nMBnPYH1nC1a1+TE4nZQC0uUth3LCQV4kaGXEZzl5i/qgDlvLgR9Dni9eGI1hMp7BBRs6AACvOJVbQkLBajRlhQMRfRrAfQAeI6L/APANAJ0APkJE31iE8S1ZHA7SM5dEM76wz42JWJqnZPrd6G8LWFoOv5f88TsHuFbz1NEZrOsMIuRz6Z0+ZxIZvO76h3HZ1x7QzfSRuZQ+2fk9TmzpDZXVHJ8ZnEGBARdu6AQA/MMVm3BKdwsKjOFfrt5iaDbW6ndjPJpGrsAqCIekZbxBsEVbWvH543yCGJxOwOty4Nw1vAPlwFQCx6aS8LudeJHmThGTiaz5iz5WYqI9s08TDlrgzq9Zb7KgEt+JjJi4xAR+cDymT8ZCOJRbBnJKE+AitVam1a8Jh5kkHAR0h7xY3xnEwFSiqh79wu0oYj4AdzHIbqW7do/gDumeGZhKYFWbH04HYfOKEPaNRKty48jus19pactHpxJIZPJ487l9AHgVNMCvx1/fuF1fjS+ayhqus8i6mYilMR3P6tfmFG2pzWa1Fdk1NIuN3S36vcpTbvlYxGpuG7qD6IvwbKu5ZBYelwNeV+l9I9MaKBUO+gpyUkp3b6vfsmpeWMZW9xBQtBzk+WLnMT43nLe2HQC/tq1+tz5nNJpKlsPbAGwBsA3ARwBcyhj7JwBXALi8wWNb8ojJSfgwQz6XblKG/W6sjPhxfDZZkm749MAsWrzcDbVrkD+MhydiOLUnhFURv36D/PZpnomULzA8dWwajDGeKy1lzZyzug3PDMzYugOe1wTNaVodRovXhT/83Uvx6D9dhled0WvYN+J3Y1irJA56LFJZtQeOCz97TWtVxI+Ax4mDWqOxoZkkVkX8+uJBx6YSGItyIRfRHjqhWcmuKoeD4HEW8783r+CfQfwv0m3lsbicpZ01e6T2B9FUFmPRtC4cVkX8cBAwVKb7q0ghbLOyHPwuzCVzOD6bQlfIC7em0RdYdWtRHNXqZFa2GtujC4H03NAs3veTHfjgz57S3WIDUjHVxp4QphNZvSFeOR47xPthvXhdO/aN8vtCWHevPasXTslf/uudQ7j7+VF87e4XAKDEmhTKQTSVw2Q8o8cgOls88LudlkqRKJQ70qBCOd4ccRanaxYmwNudDE0nkckVdEG8TrccEphL5UrSn62wshyOz6bAGA8mC3ojPL5lfuZjmoVilTAB8Pso4HEamvsJi064rYgIZ/VHanIjLoRKwiHFGMswxmYAHGSMJQCAMZYDsLhJt0sQoaUKrSPkc+mTTNjnQlvQgwIrpssJjk3FcdrKMNZ1BvHcMHe5jEfT6Al70ddWzB8/IC1Ss28khplEFulcASukieTs1RHEM3nb7Jbnh+fQ1+Y3aPoup0O3PmTCfrc+frtUVn3fMm4lIkKvlD46NJ3Eqja/wa86l8oh7Hfr55mKZ+B0UEnbZK/LgRntgdQrSc1uJakq2knlhEOqRFN3OR3oDvkwVKbjpqh6by9nOUh9oUS687GpypPg8GwKva0+PdALcPeVEA6PSzUTj2iB1WNTCd0NIYSEXYNCmd1DvDfSGatasX+UN3oTWu66zhZ0tnj070y08RAN4cxJCkIhSmRymE5kdJcbz9m3dqde93teKPeh/32qIfUZ47E0JmIZvSAVANa0B1BgXEERQeGesBerIn6euj2b1JW8clgJBxH7kmMxK1v9yOQLJcI6ls5pmXXWU654ZuQ2L7uHZ3Fab9hwb5zV14oXRqNIZRu7XC9QWThEiOiNRPQmAK3a3/r/Cz05EX2UiJgU0yAi+iYRHSCiZ4nonIWeo5EIzdWjWw5uPRAX9rv1tD9z87gjkzyItbo9gJHZFFLZPOZSOXSHfdqDxXs0HZ6I44xVrfC4HDg6GS/6taWJ/ex+7qp56qi1NnFwPG5oEVyOVr9bv6mtArvyjV3OrcTHWDSvh2aS6GvzIxJww+UgTMYzvFWGz6WfZyaRMbRDF3jdTsxok3OfLlyMbiX54XZY9OT3e5wI+1wYm0uV+PgB7icu13tpOpGB1+WwnETCmnCQ+/OIyWIiWll/GplNlmRB8Z5SvIbgqaPT6A554XE68NTRaaSyecwksrr1KCyOalpuiNX95EZv41G+0E3E70ZnC8/lZ4zh2cEZhLwuJLNc8cjkCwa/vHCrjs6lkC8wg7tE3MMyyUwev9LWXN89PIddDehgLNp5yN+tCABPxdOYjKXhczsQ8Lj0+3cylrGdsGWEMmQQDlHRvbj42YW7bcpCOFSKa6yM+PU0ccYY9o1EdRetYFNPCAVWfRbkQqh0VR4A8BoAV0t/i58HFnJiIuoHcCWAY9LmVwHYqP28D8B3FnKORiM0KaFFeV3GyVO4TGakfkuJTA7j0TTWdga1RnbpYhfUkBd9bX7E0jyl7fBEHOu7gljdHsDhibjedlrWVNZ0BNAe9Oj+STOD0wn0W/RzsUKe8AMWwkGed30VtK0VWm1BLl/ARCyDnrAPRISQz4VoKos5zYctrmE8k4fbwiUkX9P2oAdel0OfCIXlIFsbVpYDoKWzapaD00FGP3HE2k8smI5nbHvvBDxOpLJ53vRP+yxCSMj59XYcn02VtFRoD7qRzTPE0jkcHI/hzL5W9EZ8GJ5N6ZOTKBDsjVTXWDCVzWM0yv3jQqCMzaUwEeML3TgcxZbss8ksUtkCLtVSQUXmldnV6Pc49e9CniD72gIlAfWdx6YRTedw3RvPAMBTkmX2jUT1Qsj5clQ7p7z2uQgAT8YymIpn0BHk341ssVopJWZ8bgc8TodROGgCQO7oahe4tisslVnZ6pcWrkojnsnrMRyBUPTK1cHUi0pX5TkAu7WfXaa/nyvzvmr4OoBPAJDty2sA3Mg4j4FbLr2W714CCK1XWA6y1hr2ScJBulGET7GvzY/OFi8m4xl9EukOeXUNbCKWxvBsEms6gljbEcDRyYTut5Rz+YkIZ/dHsNMidXI2yTuVyj7RcojxArCcqKuZhAW9rT6MRVP6ZxcPTcjnRjSVQzSV435Wt1NvReKxCAp63cVbNOBx6tecqJjqKo/LynIAuLAanUvj4HgMa9oDhhX9Vrby9EO7oO50IqNPxmY8TicyuQIvHNTG5nNzS8Wq3YJMocAwKiUYCPSeUvEsbxMS9mFF2IfjM0ld0RDfVUfQA4/LUbF1OP98wOoOvy7opuIZTMSKLR06W7yYiGZ0d4mYiOxcjX530UcuWw69ER/mUjm91QkA/f686vQVWBXxG9JLdw/P4pXfeBCXffUBw3tq5egkF/yysG2TPqscGxH30awWkK4EEc/mE/U4QNFykBWHokJoshxSOUtrXKY34sNELK3FR0otXIAH2O1qKepNpavSov2cC+ADAHoBrATwfgDzdvkQ0TUAhhhjz5heWgVArvIY1LYtScwxB5csHPwutGpFXLIWIWctdAQ9SOcKODzBNZ7ukE83PblriT/8XSEuRERA0hzUOnt1BAfGYiXayqBNDyQ7ZMvBavKXu7TazME6PWGfwfwVAi2k9WfibiU3HA7SLQCPhUDySdfW7Sz6bHldCd/fILTshIOWf35gLGbIDAL4dc/kCrbdVWeTWd1FaMbtImTyBcTTeQSk2hB5YSSACwKzJTEZzyCbZyVuJTHZjEVTmEpk0NXi1ZIbipaD+K6ICCtbK68rITT5/raAvhbJRDxjSFHlykpad42KiWnQpujR73HqNTmywiJabIxJ6x88PTCD9Z1BRAIebOk1LpL0w4d4S5TxaBoP7CsWT9bK0ckEVkX8BjeRbjnEM5iMp/X/hUs4ls5V5VYS75GF10QsjbaA29CMcaGWA9OWI9XdnybLweNyoL89YJtyXE/KXhXG2L8xxv4NQB+AcxhjH2OMfRRcWKwu914iuoeInrP4uQbAPwP414UMnIjeR0TbiWj7+Pj8b6iFIG4qod3KE6psOcxKWoTQ/NoCHn0SEIHn9qBHf8iERhb2uxD2uTGXyuqFWuaHVCxSbi68sltcxA6PKVPIjLzJ3M7bjNAkhYASGSEtXhcm4hk93Rcojd3IiGsrBLHbxc8r+//l625n0WzobsF4NI39YzE9fVWga3txa7dGzKZiHAC8TgfSuQIy+QJapJYj8sJIAPCp3+zC+Z+/Fzc8eFDfNqW3VDBaJULb3T8WA2O8+E4E+Ke0YipRPQ6IAHZ5K0UEY7tC3qI2HeNWQqe0gFU2z6SsniCcDrK1HAIeJ+JaDyjZ/SfcnrJwPDAWw2bNf74q4jfESHYem8Flm7vRFnDjbm3J3fnAYypGK9nndiLgcWIqnsFULIN2za0ka/FWVrIVfrcTCann1WSstJFeuIxwCFVhOQA8VfzgeBxBj9PgQhas0TwJjabaCukeGLOTMto2WxhjlzPGTjf/ADgEYB2AZ4joCLjgeYqIVgAYAtAvHaZP22Z1/BsYY9sYY9u6urqq/Bj1RdxUYlKVJ1Svy6FrEXLMQVgOkYBbv7FE9o3f49QnfrEkYavfjbCfr2sgHnBz36Mz+1pBBDxlijuIycm8dKYd8vitFPBqNHSBsG70bqmSW0n4VYXAEJOOlQYnJh0R4yhaDtZBaIfNHX2KZJ6LIj2B7max6XoZt6kYN49ZjtPIXTyPTSZw0xPcIP7mvQf0icMu913cN0Jp6GrhwiFXYDioTdyyC7A96MG0jWATyOm4bie/N6fiaS3TyG0Yh37ekBdhn0tXMszCwec2FoIK9JiLZjlk8wUMTCWwTltFr68tgKjWKmI6nsHhiTi2rW3HOavbFrTU7tHJUuEgPtdUPIOpRNGtFDQIh+qmwYDHaWiIOJ3IlKQ3h7QU9TkL4WCXxioQ2W7HZ7nlsKG7xVIJW9MewLE6rIteiWqFw40AniCizxLRZwE8DuBH8zkhY2wXY6ybMbaWMbYW3HV0DmNsBMDvALxLy1q6AMAsY2xhCxM3EHFTCY1XuJUcxDVrt9OBFq/LsADQrOQzFpOSePj8bkk46Cmxbn0SHZ5JosXrKpmYQz43Tu0pLYYrV7xlhcPgNrKyHKp3K4nJVK/70Cwi2RcvrKSWssKBH0e0KxGCWF5Ny1lh3AAMgb3TTMIhUmbdCKA0x19GtnZkAdLqd+uW3p8OcMv2S28+E7F0Dn/az/+fsREOcmU4AHSHi91nxb3SKgkHkd1Ujql4Bi4H6dprR9CDiVgGqWwBfu1aCmtscDqhZzBFAh699sWs+cotZDwWloNwow1OJ5ErMKzr1GpLNEt2aDqJA5r7ZEtvCJtWhHBoPF5V8aCZ2UQWs8ks1rQHS17raOEL9aSyBV2xkhUsbxUxB4Dfc7JbKZYurZFwOAitfrchzgiUv4cEYiGp4dkkDo3HS+INgjUdQUTTuZKMqHpT1VVhjH0ewHsATGs/72GMfaEB47kD3LI4AOD7AD7YgHPUDZdpMhMarDx5m/2U0wn+kLZ4XbomMRFNw+UgeKTurmISCGuWA1AUDlacvTqCpwdmDPnjU/EMQj5XVQE3oEbhUMly8BpbaYtCNdklJv7WYw4W4/SZ3Uq6cJDcStLb7NxK/e0BvPqMFXjHi1djbYdxAjG33jbDe01ZX3eD5SBNOC0+l17f8vihKawI+/CGs1ch4HHi8UO8dkEoDaL7rMCnXQdhdbUHPPo1GtYqsWUXVnuwsnAQQXWhiXa0eHTBLa6t+D08k0LE79YnOqGgWrmVBPJ31xbwwOUgPXYhKpTXaU0FV0WK6bfiM66M+LGppwW5ApuXP/2oVlOy2sJyEG4loHg/yeuVVGs5+D1Gt5LdfSFqX2TkbDY7Ah4XIlrPp6GZJNZ3lgo6QG610dgeS5VLAzUYY08BeKreA9CsB/E3A/Chep+jUYgAalbrLuq0CJB6XU6kJU1oJplFJOAGEekay2wyqz+YIclKAITlwCePoZmkba702f1tuOmJARyeLGock/GMbS8XKwyTbIWYg52GLtDdSjPGgKUcuBRWgZh0LOsctH18unDg55WzmIxuJetxOR2Eb7/jXMvX2gLF1uBmcvkC0rlClZaDJPi8Lh6L0PoVbexpgdvJW4iIAjM7t5L4rKI+xu9x6sHu4ZkkwtrELYgE3EhlC0hm8rYFXVPxovsI4NdcxBb82rXUz5vI6H/L7iuzhm10KxndfF0hr+5WEpOYKIIUrp3peEb/jCtafUhry8EekmpzRmZT+NubnkImz3D9287WJ0YzVqsTyuPUrXPt+ricDnhdPF5UtXBwO5GUis+iKWtXkRAO2XwBB8dj2NjN60qqKbbb0NWid0Ywx8YEvdK66KKbbyNQXVkXgLAcdOGgPbAuyfHtdTkM1YwziYzuUxYTSyyd028ct9MBn9uhF8OE/S5d6z4+m7L1W569mt8kchO+qXi64tq4MrJ/02rud1QR+BWIiXJ4NgmiokvCWEgl9i0TkHZZWw7yZFRpLJUI+9xwkLXlEE/z785WOEgTi7yP+PzxdA4DUwk9nXhjdwiHJ/jaCTOJLHxuh2GSBYqfWcSqfG6HbjlMxjMImPZvr+AW469lDULI63Lobi1zMeFMMqtr2IYMNpPgtbMcAO5aEm6l47MpuBykx9jkjJ7jsykEPU6EvC49pVfO6vra3fvw5JFpPDMwg4/+0pzcWERUdlvF1/hn1a6ldN+Uc2daETBbDqnSNcsBTYhk8vjBnw7jqm/8SV8ZsJrzbF0Z1lvhvMhm4i9aXgtbnrYSSjgsAJduOfAv0ynFHAQ+t9FymE0W12eWJzhZqxA3nNvJq1DlVhV2yxme0t2Cla0+3COtajYpZWdUgyHrx8pyqBCwlhEPXjSVQ4vHpb83YJFlVAxIW6Syuo0Tl5iEZC22UnC8Eg4HIRLwWAakY5pL0DYg7SqeW/5sLb7i8qXTiayu0a7tDCCZzWMsmsZ0PGMZD3I5HXA5SHdL+dxO3Q0yk8iUTMQiZlLOB20+l9fl1DvZimssBEIik9fvTbmRoTk4Kr9mtvq6Qj49tnR8JomecLFFSNDjgoP4Ikkjs7zOg4jQEfTA6SB97Y1kJo/fPTOMt53fj8++9jQ8cXgKO45adyAWixG1BUqfD5/bqcd/ZItTt1irdLv6pYB0OpcvqRrXr4XLgUy+oHfLFWswVJMVJeJhbieh2yaRJBJww+d26OuqNAolHBaA8AmKPHXdcnAaLQd5ychkJq/7puUJTn7QRJAr7HPrxTcCu0mKiHD5aT340/5x3VKZqtGtJGf62LmNxOZKqaxOqWutzxAfKJ5ECFdzMaGMveVQP+EAcIvGatEksc3ecijVROW/ReND0RdKxDsOT8QxIykKZsSELeo7xD1TYKUaaJtFx1Azwp2pj9vi3pPvQSEoyikEfinuYZ74usPFbC1eBV6c6ER7+NkkX29daPsObb3vUc0d9dzwLFLZAi7b3INrz+tH0OPELVoLDjNjc2l0aYs/mZGtBdlKK7ozq7t/Ah7uVmKMFRvpWdwXXpcT6WxBd6uJleFcdql0Elec1oM3n9uH/36bfRkZEWmtNpRwWLJcu60fP/7L8/HGc3idnphQ5YnV53YilS1aDulcwbLdhmw5FDU5o08esPbLC166sQupbAFPa/UOc6msIaulErVkI1UzIQsXmFwc6DJMNkbhYJmtpE1SYvLRYw6yW6kOwiHgcekuJBmhvdsJB3lSlIvghEYpKl2FAiHSOY9O8uVY7QLdegqvxT1gvk5ByYVlh6yUyMcHJLeSRQyhXJypnFXRrRVuZvMFHJ9NGZpFAkW/fDSVMwjInrBPdyuJup2z+iMIeFy4dEsP7tw9atmBeCyaQpeNpi1bC/KYhaJVvVvJhXyBIZMv6JaItXDglsN+LSVY9CurxnLoaPHiK285C1edvqLsfr1VFD4uFCUcFgAR4ZJNXfqD4XIYfwOllkM6V9BvViLSJ3v5ptXdU9q34zTEAuxvsPPXtcNBvDVzvsAMqXvVUClbybhv5eOJOINdewuhSZUz7y/Z1I1IwI0z+rj/1Zw+XM1YqyHocVq2boiXmQTM47CyHMx1HqIGYCKWQTKTL4k3CMyuHtll5TZdJ/Ga7A83k8zm9WMBRuEq3u+VhYOwHMq4Gu2sWKBYJT0eTWNkNmVoMw9w4TCXLF1Yqifs1d1Ku4fn0Nvq06/Z5Vu6MRXPYPdwaS3EeDRtWTAGGK0F+W8hLM3X0w7xjCYzeV1psIoBel0OJDN5vQ3JpPbbnN24ELolt12jUMKhjlilsnrdDpPlkDc8mEKDkx9+8XbxYMpzX7l5sNXvxrrOIPYcn9MnumCZB9iM00Krt6NSKis/dwXLQQ9I22crnb+uHU//65X4q4vXGfapt1sp4HXp1b4y+nW0EbJiPEQmrdSUdSYsCVGxOx3nNQZ2wsFryiByOx26IDK7QYrNC43C7ZO37sJnfvsc0rk88gVmtAwkQeGzcivploN9kkK5vH0xoe8b4R1dzf2jxAp65nTQDmkti8HphCH76EJtRbRHDpauzz1WRjjI94osIMvdd1bIQljvVmBxDTwuhyE5QBg6rjrcpwJRgd/IQjglHOqI02Iy97mcRsshWzDcrOZJAJCEjIWbqtKkvb6rBYfG47oWGajJcpA+i82NTPq+lW908TntLAexvZbAoLllCbDwbCVAsxws3DJCsNulIYoxy72egOKkISwHOZGgTQt+p0zavIzPlMIrxgiUukHEpCVX7yYzedz0xDH8+NGjeFhbElT+DFbxLrezuJ6GleVg/s7LCQcxUQsXZ6+NWymeyRsUmJBUHzI8Y+xY2x3yYX1XEE8eNgalM7kCpuIZ3VoxY2c5lGvbYoVfFg4VLAfx/MkCq1r3VTV0tXiRzhX0pIJGoIRDHbFahazUcjAJB1MFMFBqMRj9vuXHsL4riKOTCb1JXy2WQzktUcD0fSsfT1wPORBnZUUEa/D9ikOZ8+oXCq9+LbUcRLWunb9Yb+dhus6ilcbIbAoOMrb7aAvytbpT2bzlsqaArDSUFtmVCgcRcyiOf4+0rvT+0Zh2TOu6BDHpEUlJBFXEHMp1Ge0O80nx2UEhHEyWg5+vfZI3LUkb9rmRyfOajZE5YyAbAF7UF8Ezg7MGjVlfdCdc2XLwWwSka+mtBHDBK+odAhZKgyxseiXhZjU/zBfxWRvpWlLCoY5YadM8cyGPqXgGc6ksMrmC4SH1mLJxgNJiOkP9AcrfYBs6W5DJF7BvhE8ItVkO5VNZ7fa1QwgFO3eVOSBdjQYn5gSP5CFgIAAAIABJREFUwa1U8W0VCXqdJW4ZAMhoNSx2rgfZcpAREw5Pd3QbvkPR6yeVs3cr+SxSSe2uk9NB8LkdhpjJbqlHkejwaQw4W0+YQhjpzSTLZCtZTYyCzhYviIBnB/k4ek2TvN/tLAb7pWsn3G8Hx/kKdGaL48y+VkzE0oZgrKjEtnUr2bjTaq1zEPvlCgVJabAv3ARgiLVUk61ULWINiUYKh+pnDkVFxH0iz5tetwNzqRzO+dzd8LkdXDhID6aYdOR0z6LFINxKxeNVur96tJtRtCCwWgvafvy1BKSrEA6mDCPAaDmY3UrVPKQFTToYYw4Lf+gCHhcSFtlKosDRTnAVLQfjoyS7usy58G0BD45NJbSAtI1bycLdqNd6WFyngMdlEG5DMym4nYSwz6036zMEpC1iDvLf4vqSZMWakyHKWQ5upwPtAQ8m4xm4nYROU72NfE9YFQ+KxWzMloNYH3rvyJzuchJtw23dSi7rz1prnYO4X/MFpt8XVveswXKQhFu1Fko1dFl0vq03ynKoI3aWgyCVLaDAjBObmOzkFszmlFhj5XL5G0zkvItAqNWKbnZUG/gGqgsCW1kOVn+LCaGaBmgiuFfPCmmAC9FMvqA/9IJyGiIfR2lCAWD8nObCxbaAW7McyriVXE7Db8AYGzAT8DgNwk0Uva1q8+vptFapqvJnAErTqM3JETKVGsmJCWxDV0uJ6092sbQYYg5abzFtKdhWv7FOR7SGkZfJ1C0HG7eSzyLQLo+/WstBKDY5STiUq80BjO60emYr6c0iG9h8TwmHOmKVjWClGcoPpvBzy5Wd5lRWQH5Iy49BCBnRVK2c6W+mlpXeqoo5WLQTsRIOQrOuRjgIt5I8udTHrcTHYI47lNMQ5e1mLZqISj6foDXgQTSVA2NGl4eM3nDQU2o5WI0laIqZTGpLm3aHfHrPKDu3kqxwiO9MvG5lvernrNhllGvNGy3WMJfvCfk44lod1xcRMlldQb4OirxY0Fg0DSLYFnzKn9VgsZSxxKwQ32ehwPSuB5Ush85QcUzuOmYrRfR+YEo4nBBYadOyIBDIN4/wE8vLUJrdSvLfldw5kaDJcqhBONSSFVXJggGKE3glyyES8OBLbzoTrztrZcVjikBkLWOtBhFQNtc6ZLTWKPYBab7dykITn8/c1tln49Ix7iMCwxZZRRZCNGCKmYi+WoYMJRvhICNateiWg8P+vqvksrxkE19nxWrylZ8BS+Fgs/ocwDsTHJQsh/EoX+HNTjOXr7F839ZsOThly4HfF+WaRQLG57qeloPb6UDI5zKsFVNvlHCoI+JBkoPG1pZDcVu8nOVgkT1UaRoMaes96Kt3zbcIroKWU51bSbMcnDbCQTrftef12/aSkRE5KvLZ61UhDaCkSjqTK8DjdNgKQzHJWRWEic9vTgqQJxQ7t5KY0ORiYLHNOuZgbAo3nciiPeixDTwLQWH+WF2aa0ZfF13cdxYfv9Jk95Ztfbj6zF68/5L1pe+VvjNjN1vjSohWcY0NXS26qwyAYTU7K+wEYS2JEEAxtpWvwa3UbhAO9bMcAB67UpbDCYKVW8nKcpCDgcKnLXdPtTLlySL+YAURX6RFCJ1ATams1n9X2tcOMXnIk7dVQLoWRIzG4HKri3Cwthyy+ULZQGJxfQl7y8GsmXoMPn7rR/DVZ/QCMNXMmFqJGMdv7A01GePatF0BmNhuLuLq0YK6omZC3IvzqbUKeFz41tvPsXYrGdbBKG07IvoGWSk3G7qDmIhl9IWz5HWwrRDX27xPf1sALgeVBL3tkGMOmVwBDrK+h81rWwjcdcxW4sd2W7aZrxcqW6mOWAeky5udAtn8FMexSiOsxoXSGnBjMp6B00FV+1PN56tPKisZfgPWRXC1ICapWtqHV4OYrHKmvj2ZXKGsZulxOmzX+rX6/IBRWNhZDueva8fNf3Mh1ncFS/a1Go+81kA2z4uj2oIezCWLAsOqCC5savwngroiyFvpO3I7CZdu7i67j937BFYupmiKV05bCf712opyBydiOGd1GyZiab1nlRXimG8/37js/eqOAHZ99pVVrbMAyNlKBU1pKJ+kABhX7Ku35VDNCoALQQmHOlKsSyhus6ugNCN3zCya8lYxh8rj4NpKHF6XvTvECqvz2VFLKmsjLAerIOpC0LXCvFE4lJsEAC7sbvvIS/XGejLCDWGeFOTj2cUcAC4gZITFaZUv73KS3pBOrF8c8bsN7eKt+neZM6ku2dSF636/Fxeu560qil14rce4//Ovth1/OQxrnkidbWWhYbew1QZtydeDYzGc3R+paDls6gnhzr9/GTb1lC67Wa1g4GOWLIe8vdJg1fEWqG8qK8Ath0MTsco7zhMlHOqI1WTXbtGvX7Yc1nYEcGQyYZgwxHGs1kauZq4XZnqtGnVtvZUqH09MAPJEUEuhnRXCcpDfWQ+3klN/8E2prGUmAYGd1iomk3JuJa+NW8n6eHxfKw+Pk4rCIaUJBJ/baZmmChTbgrSblijd0hvGvv+4SurKOn+3UjnsLAeR5ZUvMNs6ir42P5wOwtHJBGLpHNK5gl4UZofdqmq1INc5iFiUFfLzbazxqa9bKRLwNDQgrYRDHbEUDhbpdfKEcOsHLzKsfAVIVoJ0L5HptXKISclZo6ZiFEbl961pHLLlIAen5+EO+shlG7FvNIqXbewqHqcOwkE8xOZ20OUmgUo4LT4/YGycZ+dWssIluTVKXpMsh7TmXuLCwTpD6ZzVEbzzgjX44Cs2lBzL0JqkvsqujiwwzRq1LhxsLAe304GVER+OTSX0CmE5ZbRRiMk9l+cB6UqWQ8DjNFq4dbYcWrw8zsQYq8lDUC1KONQRvfGetM1SOMjZDFretoxV+mDRvK98E+jujBqfbPnQ9XErlY6j0mpzlTh1RQj3/OMlNY+lEuKamWMOldxK5dB7S5kmBY+NNl8Jh+TWMCMmVKBoFXhdDvt6BqcDn3v96ZXP2YBJh5/fOBbDaw5CBvarHgLAmvYgjk4lMBHjPveuluqCygvBKSkQmTJrT4skk3PXtBm219tyCHpdKDDer62W+6haVLZSHbFyb1jVGVgFpA3HsQg+FwVG5XGIe7bWB9uue2qlfe2olMpaL22nHpaDfcyBVZ3qaDcuc5ZKtTEHuzHm8xbCgUgXGqILsM/tlNYOqWHgErW4M2uh3EQprpt5rWyZ/vYABhbdcigK52ye2cYQzlkTwbsuXIOvXnuW8f11thxEbU6szCJPC0EJhzpipalbTYDlMisAuVV3cVstD6lrnpZDLcKkulTWUguoHhO5mXpkKzlNLptMroBP3roLB8dj8w4kWmWdAdXVOZQdo0UAwOlwlFoOboeuiMw3jVIMvZExBzN6rKaMUF7TEcBUPKP3EKsUc6gH8j3CY1H2rU/+/ZrTS3o91TuVNajX5jRGOCi3Uh3Ri+BsJqu3nNuHF6/vqKiJksWkUksqa7H9Rq2WQ/X7VlUhbXH+RgiHejxzbqn6FQAePjiBm544BgCWmUjzObb+fxV1DlaI61mwcCsZYg6a5eB1FQPS89VaG/F9AeUL6Kpxi67RFgHacXQaTgcZ6gkahbnOodq1p/X3N8hysFreth4o4VBH7DRYt5OQzTN8+S1nWb5uxiqVVUQyapmUa485VL9/Ndq61QTQEMuhLtlKxWAjYGxoNl+3kmj1Yb4Onnm6lZw2tRiAdczB5y7GHOZ7jRoR6ASqsxzK3b/9knDoCHrqkrFWCXNX1lpjUfUXDtYrANYLJRzqiN3N/NSnr4DF82yLVSprsd9S5fdbLVda1XlrcStVlcrKjye7JBohHOpa56B9UaI3FVB9Y7ZKx9aPZ1jsqfpjr9fckZssKo55zIELBYPlIC0zOh8aFZAuNx5xj5SzLlZ3cOEwm8zi9FXh+g7OBuGuTecKZbOV7Ki7W8mr3EonDHbaS7msCyuEtmZ1L1Va7AewTiGthtpiDtWPg0mZ+fXO2Kh2LJUQ1+rBF8bxxOFJg0Cb78QqDlEiHKTj1aKZX3RKJ3734YtwhramgYzTQSgw4LX//ZCeGu1zO/TGffMVynXsFWegnEAvriBov0/Y50Z7kC+atLW39Ho0AnENv3znPgDFxoLVUm/rJmjTD6xeKOFQR/QK6QUfh/+2mvSqy1YSwqG2J7ua3a3aV9hh6VZqgCZal2wlbUL63TPDAIzVyfN3K4ljm7KV5nk8ADizL2K5XUyku6QV4AyWw5JzK1VjOZQ/twhKn7ZysSwHU+yoUZKzSvSYQ4PcSipbqY7Uy2VSrj13NdrHfGMOtWjgtaSyGtxKdfa7AvWxHMwWzYS0/OJCJ4FylkO9sLov5JjDfNtF6xXS8x+aJeUmfrvKcjMfu/JUALVr8PPFfI1rcQk2gkZnKzXt0xHR3xLRXiLaTURfkrZ/kogOENE+Inpls8Y3H+qlFZfro1TNKfSaiJqzlSrvX6nXjmEcFjstVcvBfAx5+UWPa37HryYgXS+sF5pyWtaa1MJiVEibERZvpe/1olM6ceS6q7G2Qmp4o6h3r6RaOSljDkT0CgDXADiLMZYmom5t+2kA3gpgK4CVAO4hok2MscY41eqMfjMv8J4pts+wcis1Mlup+n2r0ta1XWStc6lmK5mvVTRVfODmO5mLz22eRObrpiqH+Ro4iH8msSjNfIOh9XKVmik3HnG567lyWiOYSTaur1E1eFwOeJwOvT1/vWlWzOEDAK5jjKUBgDE2pm2/BsDPte2HiegAgPMBPNqcYdaGXqi2wOOUizlUc+xqNa+S99XZrVTP95WjLm4lCy1wVcSPF69vx5vO7VvQsc2fuREap1m4+dy8r8+6ziB6wl586uot8zpuo2IO5SyZYvHg0vZ6H5tKVLXff77hDDx+eLIhY/juO8/Bmo7GWE7NEg6bALyUiD4PIAXgY4yxJwGsAvCYtN+gtq0EInofgPcBwOrVq612WXTqlcesPxxyKqvptbLjsEiFreW85SgGpGs6tHSO+b2vHPVJZS2diLatbcPXrn3RvI+pB6RNx67ncpECswAS/nC/x4nH//nyeR9Xr5Ce9xGsKfesWK0TvtT4s239eOeFa6ra9+0vXo23v7gxc9Slm3saclyggcKBiO4BsMLipU9p520HcAGA8wDcTETrazk+Y+wGADcAwLZt2+p9786Lek18xZhBcZu+PGZN2Uo1Coca9q9FW5cD0o3QROuRImh1iIUGokUK72L4ps1adr0asTUiRgSUdyst5nWbL+96yRpsXbk4KbTNomHCgTFmq64Q0QcA3Mp4xO4JIioA6AQwBKBf2rVP23ZCUK/+M+VaZdTSPqPW57qaOZaIf85qxrFYj3Y9XFVWQqtesYFGtaCQKVdotxCoTq5SM9VYBUvZrdTsTKXFoFmf8DcAXgEARLQJgAfABIDfAXgrEXmJaB2AjQCeaNIY581CteNi4z2LmEMNlkPN563ifbXUOejvqbtTwkijtNuFZhWJa7UY+fBm66kerjagcW6lctekeN2WruVQqbPyyUCzYg7/D8D/I6LnAGQAvFuzInYT0c0AngeQA/ChEyVTqZ7oFdLSszGfmEOt1FQhXcV8pwvJBjv9GqVgLlT7LsYcSq/rJZu6cPEpnQs6voz5HJ116lLaKKunGuHQiGr6etGIjLOlRlOEA2MsA+DPbV77PIDPL+6I6kO95sByk3QtFdK1urnq3T5j0dxKTWgOVwtWLpQf/+X5dTm2wDyJ10s4NKq3Ui1FlEsR5VZSzIt6pbLKFAPS1cccaqWWt9VyjkZnC9RbuxWX2OOsj+tgMTTgUsuhPi2s9aLHuhytlL+7bGPJNr0n1RJ2KynLQVETp3S14N0XrsG7XrJ2QccpJwCqW+yncTGHYkC68vEapHRanKe+Jwp6XIilc3VwK4kK6cZfCHPM4TypN9SCjtug9hkAcOS6qy2321WWLyUaUeW+1FDCoY44HIR/u6byuryVsJqka4k5zDfLo5pJVriqqtn3Vaf34p49o/jEVafOazy1snlFaSvr+eBzOxFL5+rnVlpky+H2j1xctzTLxci0smMpu5WWsuCqF0o4LEHKPRPVPC+L8VBV4+f3e5z49jvObfhYAOC2v70Y/W2Buhwr6HViIrZwv/JiukfkSbye+feLZf1ZsZSFw3JACYclSDnroBqNfTFWxWpUoHK+nG6xxsF8CWjdLhdcBCdSMhfBcmiUht/M77nZLbGXO+rqL0GsHkg9IF3F+xdD41rCWYYLJuDhgegFxxy0b60RbcrNzLfwsdrjLiZCqDbTpaVQwmFJUu4Br6VCupEsNcuhnvjc/LGol+XQqFRbGRHXqPf30oz5WQjVpZyttBxQbqUTBD0gXcvazQ1MIl3IJPQvV29BV6g+efiNQAT0F245cBZDjjaqtXajurJWg3IrNRclHE4wlozlsIDn9r0vranH4qIjhOuJZB05GzTm/7+9uw+2o67vOP7+3OSGYEIIENQUCEFAESiGJCIPDkalTshQUjU62PIUH1JbIWqHUVs7FrTVGUY6FTtDyiDGUAFB1AkRBh8gBYGgEUkQkZoiHVFGHtqGplY6Id/+sb+TrHfPPffc3Hv24ZzPa+bM3ad79vs7e85+d3+/3d9W8Rm4WqkenBwaooyb4MajSTvO8Wp9fi/umtiZ15pzF3L13Y9P2t3KnUztVZtDFckh/S2jIX+81py78HceBNXPnBxqqNPPsZufahnJoYqdRlmmTlJyWHT4gfzTeZNzM9pYenXmUOmlrDVsc1h6/NyqQyhN/VKztW0pGN9NcD5zmIjW57dz166KI+ler65WqnI7u1qpWk4ODdNVtxUldHmnPv7mLDhsNpA9JrQpenXmUM2lrNnhkVNDtVyt1DBVXj2S14/VSn955jE8u+MF3vP6IzjlyIMa9aSvXrU5VHnw3odfsUZxcmiYuvxg+rFa6U/fcOTu4SYlBuhlm0N6vxIfxFuLZ/6aq5Wapi475RpeSDLQ9iSH3rxvqXZnh3p81weVf+INM57f6mQ907p9HP7h1kmnR8tOxO7vW4mb+yNLX8U+U4ca1ebTj1ytVEOdfod12Sn3Y5tDo7UeytMHVystPX4uj/3t4FwyWlc+c2iaEn6rnfYHl519HNOmDNWm7cMy43nOxnjsfjs3BAwcnzk0TK+P5L730Tey7/Doj8e84NT5XDDBJ93Z5Nt/32EAVrd57OZE+F6DweXk0DDd/FYPmJHtKI566cxxv/+hk/TAHCvX9OEpoz52cyJ2H4w4RwwcJ4eG6ebM4bjf25/r3/s6Fs0/oISIrJ+5WmlwOTk0TLcHcKceNaencdhg8IUHg8sN0g1TlzukbTDU5eo4K5+TQw11OoN3+6CVyclhcDk5NMyQs4OVqJ87WLTOvOlrqPNNcKWFYeY2hwHm5NA4/rFaeVytNLgqSQ6SFkjaJOkhSZslnZSmS9KVkrZJ2ippYRXxVe30Vx4MwPITDynM85mDlcm5YXBVdSnr5cBlEXG7pGVpfAlwJnB0er0OuCr9HSivOHjmqDc0+UjOyuSv2+CqqlopgFlpeH/gV2l4ObAuMpuA2ZLcA1eOk4OVaXhoiCHBX5/16qpDsZJVdebwIeAOSZ8lS1CnpumHAL/ILfdkmvbUyDeQtApYBTBv3ryeBlsnzg1WpqEh8fhnJr9bDqu/niUHSd8BXt5m1seBNwMfjohbJL0T+AJwxnjePyKuBq4GWLx4cd/f3L+7ixsnBzMrQc+SQ0SMurOXtA74YBq9GbgmDf8SOCy36KFp2sBrdcnsaiUzK0NVbQ6/At6Qht8E/CwNrwfOT1ctnQxsj4hCldIgc3IwszJU1ebwPuBzkqYCvyW1HQC3AcuAbcBvgJXVhFdfzg1mVoZKkkNEfA9Y1GZ6AB8oP6L6ayUF3+dgZmXwHdIN415ZzawMTg4N4QZpMyuTk0PDODWYWRmcHBpiT5uD04OZ9Z6TQ8M4N5hZGZwcGsYP+zGzMjg5NMSeBulq4zCzweDk0DByk7SZlcDJoSHc1mBmZXJyMDOzAieHhoi+75TczOrEycHMzAqcHBrCbQ5mViYnBzMzK3ByaIiXTMt6V/cZhJmVoaqH/dg4rV35WjZsfYqX7rdP1aGY2QBwcmiIww+awQfeeFTVYZjZgHC1kpmZFTg5mJlZgZODmZkVODmYmVmBk4OZmRU4OZiZWYGTg5mZFTg5mJlZgaIP+oKW9Azw73v573OAZycxnCo0vQxNjx9chjpoevxQfhkOj4iD283oi+QwEZI2R8TiquOYiKaXoenxg8tQB02PH+pVBlcrmZlZgZODmZkVODnA1VUHMAmaXoamxw8uQx00PX6oURkGvs3BzMyKfOZgZmYFTg5mZlYwMMlB0lJJj0naJuljbebvI+kraf4DkuaXH2VnXZThQknPSHoovd5bRZyjkXStpKcl/XiU+ZJ0ZSrfVkkLy46xky7iXyJpe+7z/0TZMXYi6TBJd0n6iaRHJH2wzTJ13wbdlKG220HSdEnfl7QlxX9Zm2XqsS+KiL5/AVOAfwNeAUwDtgDHjljmz4E1afgc4CtVx70XZbgQ+MeqY+1QhtOBhcCPR5m/DLgdEHAy8EDVMY8z/iXAhqrj7BD/XGBhGt4P+Nc236G6b4NuylDb7ZA+15lpeBh4ADh5xDK12BcNypnDScC2iHg8Iv4PuBFYPmKZ5cCX0vBXgTdLUokxjqWbMtRaRNwN/EeHRZYD6yKzCZgtaW450Y2ti/hrLSKeiogH0/B/A48Ch4xYrO7boJsy1Fb6XHek0eH0GnlVUC32RYOSHA4BfpEbf5LiF2r3MhGxE9gOHFRKdN3ppgwAb0/VAV+VdFg5oU2abstYZ6ekKoPbJR1XdTCjSVUVJ5IdueY1Zht0KAPUeDtImiLpIeBp4NsRMeo2qHJfNCjJYVDcCsyPiBOAb7Pn6MPK8SBZXzWvAT4PfKPieNqSNBO4BfhQRDxfdTx7Y4wy1Ho7RMSLEbEAOBQ4SdLxVcfUzqAkh18C+aPoQ9O0tstImgrsDzxXSnTdGbMMEfFcRLyQRq8BFpUU22TpZjvVVkQ836oyiIjbgGFJcyoO63dIGibbqX45Ir7WZpHab4OxytCE7QAQEf8F3AUsHTGrFvuiQUkOPwCOlnSEpGlkjTzrRyyzHrggDa8A7ozUIlQTY5ZhRN3w2WT1sU2yHjg/XTFzMrA9Ip6qOqhuSXp5q25Y0klkv6/aHGCk2L4APBoRfz/KYrXeBt2Uoc7bQdLBkman4X2BPwB+OmKxWuyLppa9wipExE5JFwF3kF31c21EPCLpk8DmiFhP9oW7TtI2skbHc6qLuKjLMqyWdDawk6wMF1YWcBuSbiC7kmSOpCeBvyFrkCMi1gC3kV0tsw34DbCymkjb6yL+FcCfSdoJ/C9wTs0OME4DzgMeTnXeAH8FzINmbAO6K0Odt8Nc4EuSppAlrZsiYkMd90XuPsPMzAoGpVrJzMzGwcnBzMwKnBzMzKzAycHMzAqcHMzMrMDJwWpBUki6Ijd+iaRLS45ho6TFafi21vXoE3i/JZI2jDI932vodyayHrNecHKwungBeNve3sma7iSdNBGxLN3B2iv3RMSC9DojP2Oyy2K2N5wcrC52kj0/98MjZ0iaL+nO1KHgdyXNS9PXSloj6QHg8jR+laRNkh5PR+jXSnpU0trc+10lafNo/emnZZ6QNEfS+3NH+D+XdFea/xZJ90t6UNLNqa+f1jM3firpQeBt3RZe2bM41ku6E/iupBkp9u9L+pGk5Wm5fSXdmMr0dWX9/bfOdnbk3m9Fq8zprtxbJP0gvU5L0y9N69iYPq/Vuf8/P33eWyRdJ2m/VP7hNH9Wftz6UBX9hPvl18gXsAOYBTxB1pfMJcClad6twAVp+N3AN9LwWmADMCU3fiNZn/nLgeeB3yc7CPohsCAtd2D6OwXYCJyQxjcCi9PwE8CcXHzDwD3AHwJzgLuBGWneR4FPANPJetM8OsVwE22eK0B2l/V24KH0+jjZ3exP5mL7NHBuGp5N9tyCGcBfkN0dD3ACWVJtxbwjt44VwNo0fD3w+jQ8j6zrCYBLgfuAfVKZnkvlPC6tb86Iz+uLwB+l4VXAFVV/b/zq3cunr1YbEfG8pHXAarJuD1pOYc9R+HXA5bl5N0fEi7nxWyMiJD0M/DoiHgaQ9Agwn2xn/E5Jq8i6j5kLHAtsHSO8z5H1cXOrpLPS/9ybuvCZBtwPHAP8PCJ+ltb5z2Q70XbuiYizWiOSLiTrvrn1vIi3AGdLuiSNTyfbsZ8OXAkQEVsljRU3wBnAsdrzSIBZrTMd4JuRddb4gqSngZcBbyL7XJ9N62nFdA3wEbJeTlcC7+ti3dZQTg5WN/9A1uXyF7tc/n9GjLd6pd2VG26NT5V0BNlZyWsj4j9T1cv0TitIO+7DgYtak8h25O8asdyCLmMeTb4sAt4eEY+NWEen/8/3hZMv0xDZ08Z+2+a98p/Ri3TYJ0TEvamKbwnZ2Vrbx6Vaf3Cbg9VKOkq9CXhPbvJ97Ol87E/Iqnf21iyynfB2SS8Dzuy0sKRFZMnk3IjYlSZvAk6TdFRaZoakV5L1rjlf0pFpuXcV3rB7dwAXS7t7Fz0xTb8b+OM07XiyqqWWX0t6taQh4K256d8CLs6VaawkdifwDkkHpeUPzM1bR1ZN1W3ytoZycrA6uoKsDrzlYmBlqkI5Dyg8VL5bEbEF+BHZjvx64N4x/uUi4EDgrtQofU1EPEPWRnBDiul+4Jh0ZL4K+GZqkH56b+MEPkVW/781VYl9Kk2/Cpgp6VHgk2RtKS0fI2uDuQ/Id7O9GlicGph/Ary/04oj4hHg74B/kbQFyHeN/WXgAOCGvS2YNYN7ZTVrMEkbgUsiYnNJ61sBLI+I88pYn1XHbQ5m1hVJnyerhltWdSzWez5zMDOzArc5mJlZgZODmZkVODmYmVmBk4OZmRU4OZiZWcH/A09HDwZiAAAAA0lEQVQgq1zuqXC/AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import scipy.optimize as opt\n", + "import scipy.signal as sig\n", + "\n", + "ntaps = 64\n", + "N = 4\n", + "\n", + "#optimize for 16 filter coefficients:\n", + "xmin = opt.minimize(optimfuncQMF, ntaps*np.ones(ntaps), method='SLSQP', tol=1e-8)\n", + "xmin = xmin[\"x\"]\n", + "\n", + "err = optimfuncQMF(xmin)\n", + "print(err)\n", + "\n", + "#Restore symmetric upper half of window:\n", + "h = np.concatenate((xmin, np.flipud(xmin)))\n", + "plt.plot(h)\n", + "plt.title('Resulting PQMF Window Function')\n", + "plt.xlabel('Sample')\n", + "plt.ylabel('Value')\n", + "plt.show()\n", + "\n", + "f, H = sig.freqz(h)\n", + "plt.plot(f, 20*np.log10(np.abs(H)))\n", + "plt.title('Resulting PQMF Magnitude Response')\n", + "plt.xlabel('Normalized Frequency')\n", + "plt.ylabel('dB')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAEWCAYAAADy9UlpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nO3deZwcZZnA8d/Tc2Yyk5kkM5lJJjfJ5CCBQEIAQcADBeTwQA5RjOKyuqJ47Srigcequ+t6gAoiIiAIciorUTmU+0xCIPdMCAk5JpPJNWfmfvaP9+2k05nuubqnumee7+czn+muqq56qrq6nqr3festUVWMMcaY4SAUdADGGGPMYLGkZ4wxZtiwpGeMMWbYsKRnjDFm2LCkZ4wxZtiwpGeMMWbYSHrSE5GbROSbyV5OMoiIisgM/zrueojI10XklsGLLrWIyAgR+T8RqROR+4KOJ5FEZLOIvDvoOGJJ9G9MRG4Tke8nan4R8y0VkadFpEFE/jfR8+9HPGtE5IxeThtzHxCRM0RkWwLjukxEHk3U/BJFRJ4UkU/FGDfVHy8zByGOAS2rx6QXeeCPGHadiNzZmwWo6qdV9Xv+cwPeOUSkQkTuE5Hd/gD7uoh8SUQyBjLfnvS0Hqr6A1XtdodIJBGZKCIPRKz/ahFZkuzl9sKFQCkwVlU/HD1SRIpE5FYR2ekPepUi8rXBDzOxfIJQEbkgavhP/fAlyY4h0b+xJLoS2A2MUtUvR48UkX/3+3ODiLwpIv8ea0YRB76lUcPvFJHrehOMqh6tqk/2bRWST1XvUtX3JGPeInKqiDzvjx17ReQ5ETkhGctKVWlVvCkiRwEvAVuB+apaCHwYWAQUBBnbIPo9bv2nAGOBjwE1gUbkTAEqVbUjxvifAvnAHKAQOB/YOEixHZSkM9FK4PKoZVwEvJGEZaWzKcBajd0jhuC242jgLOAqEbmkh3meKCJvS2CMgUrmlZKIjAL+AtwAjAHKge8ArclaZkpS1bh/gAIzooZdB9zpX58BbAO+DOwCqoFPREx7G/B9YCRwAOgCGv3fBKAZd3UQnv54oBbI6iaWO4FHeoj3fGANsB94EpgTMW4z8BXgdaAO+COQGzH+3338O4BPRq57L9bj4DYZaBw9rF8jsCDGuDOAbVHDNgPvjvje7vPbsQFYBVQA1/jvbivwnjjLnuPXZb9ft/P98O8AbUC7j++Kbj67Gnh/nHmfCaz32+MXwFPAp6L3N/9+qv9uMv37TwDr/DptAv41epsAXwV24k4aQsDXcElpD3AvMCbiMx8Dtvhx10Zuw27ivg34Me7EY7Qfdi7wV+BZYIkfdhTwDz/P3cBdQFHUfv+qX4f7/D7x/QT9xm4Lz6u7/QQ4Dljhl/1H4J6o6c8FVvrv/XngmDjf49uAV/z3+ArwtogY2/1+0hhre0bN63rghhjjwvvAV4F/Rh0jrutN7Bz+2xgB3A7s8/vSf0Rto83E+M1GfD9f99/tZuCyiM8WAnfgjmtbgG8AIT9uCfAc7qRwj/8elwDPRh2DPw1U+fX4JSB+XAbwv365bwJXEfHbiNpmi4D9cbb3dcT/nT0J/BB4GagH/oz/3URMeyXu+FkNfCViXouBF3z81bjfeHYf1vHHfh03AZ+NimuJH97gt8FlsdZRVRN2pVeG+2LLgSuAX4rI6MgJVLUJOBvYoar5/m8HbkNeFDHpx4B7VLW9m+W8G7g/VhAiUgHcDXwBKAGWAv8nItkRk12EO4ucBhyD22CIyFm4nfpMYKZf1hHirEdC4uiFF3Hb9xIRmdzLz0Q6D3fgH407yP4dlwTKge8Cv+7uQyKSBfwf8CgwDvgccJeIzFLVbwM/AP7ot8dvY8T9nyLyCRGZGTXvYuBB3MGgGJeMTunDOu3CHdxG4RLgT0Xk+IjxZbgz2ym4H+XngPcDp+OSwj7cjwwRmQvciNsPJ+Cupif2sPwW3AEgfFVyOe4gd9hq4g4YE3AnD5NwBxn8fvEQLjGMwe07H4j6/EB+YzH5Zf8Jt0+MwSXcD0WMPw64FfhX3Lb4NfCwiOR0M68xwCO4ZDUW+AnwiIiMVdUluET/3z6ux3uIS4C3406u4vkVUNFdfVtfYge+jTtwT8cdAz7azTTxfrNluH23HPg4cLOIzPLjbsB9d9Nx+9zluP007ETcQbsU+M8Y63kucIJf7kXAe/3wf8F95wtwJ07vj/F5cCUSnSJyu4icHb3/9NLluAuC8UAH7ruO9A7c8fM9wFcjvpdO4Iu4bXQy8C7g36I+G28dz8WdnC3CVaUAICIjfQxnq2oB7qRrZbwVSFTSawe+q6rtqroUdyY3q4fPhN2O38F8vdyluB9gd8bizhJiuRh3JfiYT5o/xp3B/UlEduEOONer6g5V3Ys7iC/wn70I+J2qrvYHj+viLOcJYJyIrBSRh/sQR2QxTKw4evJh4Bngm8CbPoa+lMk/o6p/V1cMeR8uKf/Ix3kPMFVEirr53Em44skfqWqbqv4DV1RyaS+X+zncQe8qYK2IbBSRs/24c4A1qnq/j+NnuKuyXlHVR1T1DXWewiXmt0dM0gV8W1VbVfUA7ozyWlXdpqqtuO/6Ql+0dCHwF1V92o/7pv98T+4ALvfb7nRcIomMcaPfH1pVtRaXEE73o08CMnH7RLuqPog7m440kN9YPCcBWcDP/Lzvx12hhV0J/FpVX1LVTlW9HVccdlI383ofUKWqv1fVDlW9G3f1fl4/4roOd3z6XQ/THcAliu4a3vQl9ouAH6jqPlXdxpEHc+j5N/tN//0+hUv+F/lj2iXANaraoKqbcVdmH4v43A5VvcFvswMx1vNHqrpfVd8C/snhx62f+315H/CjGJ9HVeuBU3FXSb8BakXkYREpjfWZbvw+4hj5zYh1DPuOqjap6ircd3epX/ZyVX3Rr+Nm3AnI6VHzjreOP1PVrX7b/zDqc13APBEZoarVqhr3RKk3Sa8T96OIlIX7EYbt0cPrcppxB8je+DMwV0Sm4c6w6lQ1+gd/cDm4M4xYJuCKDwBQ1S5ckd0y3BkaHH4wjYxzgp82bAuxtQG7VHWBqp7fhzjKI6aJFUdc/kf5NVU9GndmuBKX1KU3n+fw+r8DwG5V7Yx4T4xYJgBb/bqEbeHwdYoX9wF1jX0W4k5e7gXu81cHh217dWUWW7uf05H8WeuLvmJ+Py6JFkdMUquqLRHvpwAPich+P/063H5e2k0sTbj9rqf1exZ3AnEtLmkedvAS13LxHhHZLiL1uGK4cIwTgO1+vcOi138gv7F4ult25L4/BfhyeFv57TXJf667eUX/bnq9j4SJyFW4K4r3+ROPntwClIpIdHLta+yR27y7/S/eb3af31fCtvh5FuOOl1uixkVuk97s6709bsWdl6quU9UlqjoRmOc//7NeLL+7+W/BrVtxnPET4GADxL/4hmz1uJKhyM9BP47NfptfjDuRrRaRR0RkdrwV6E3Sewt32R9pGvGTQixHVGD7g9G9uKu9jxH7Kg/gcSKKXrqxA7ejAweLSCbh6lL2Rk7oG8V8FDhbRJ7BbeRJEZPEKzrs6dEUseLY3sPn+kRVd+OuIifgiqaagLyI5WbgDsSJsAOYJCKR+8xk+rFO/ozzB7g6qGm4q/eD2z5ie4Udtl64oqTwtDnAA7jtUKqqRbji5MiTgOjvayuuOKQo4i9XVbd3E0seLkn3xp24erfook1w66u4BlijcPteOMZqoDzqxGUS/dPdvhlz+8VYduS+vxX4z6htleev4qIdtt9HzKvX+4iIfBJX3/ouf8XVI1Vtw9Urf4/Dv/e+xF7N4cXYfd3+o31RW9hk3PbYjbtAmBI1LnKbDORRN/2OW1XX44rU5/lB8faT7uY/Gbduu+OMDxev34i76p/p9/+vc/h3Fc9hv0mijs2+5OpM3AXRetxVbEy9SXp/BL4hrql8yJfRnkecurU4aoCxIlIYNfwOXPn4+cRPet8G3iYi/yMiZQAiMsM3Uy7CJc/3ici7fB3Ul3HFGc93M6+bcQfHv+Lq8qYDS0Rkrj/QfTtOHNnABBF5RUS6K0PvSxxHENcU+4wY4/5LROaJSKaIFACfATaq6h5cmX2uiLzPL/cbQHf1F/3xEu7E4D9EJMvHdx6uSLRHIvJNETlBRLJFJBe4GldhvQFXFHS0iHzQFzF+nsN/cCuB00Rkst93rokYl41bx1qgwxeZ9tTc+yZc/eIUH1uJHLrl4H7gXHFNu7Nx9Zy9rQa4Hlda8XQ34wpwRZJ1IlKOazQV9gLuSvMq/71egKv474/ufmMrgXNEZIz/3XwhatkdwOf99/rBqGX/Bvi0iJwozki/f3XXWnoprn7tI349Lgbm4orBeyQil+FODs5U1U29XWHv90Auh0p0+hr7vcA1IjLafz9X9XH5AN/x+/fbcXVQ9/lSlHtx+1uB3+e+hDtBSoR7gatFpNwfA78aa0IRmS0iXxaRif79JFzx44t+kni/s7CPRhwjvwvcH1FSBPBNEckTkaNx9ZZ/9MMLcI1fGv2V2Gf6uI6f9zloNO6kKLxOpSJygT/haMX9xuJWR/Tmx/xd3MH6WVyF/3/jWses7kPQwMEzi7uBTb64YYIf/pwPdIWqxryCVNU3cJWgU4E1IlKHO8tfBjSo6gbcGfQNuLOP84Dz/JngQSKSj6tfuwhXFPZr3IHzZ7iWVHW4FlkAS0VkNe5gFjYFd4Y0A3hQRE6OirNXcXTH74jhlpXdycM1etiPq/yegjtZQFXrcJXDt+DOJJtwrcoGzMd+Hq7SfDeuAcHl/jvt1SxwZfy7cWd/Z+KKrxr9FeuHcfURe3AV4c9FLPsx3I/ndWA5EQdRVW3AJcl7cfvnR4Du6lkj/dxP86iINOB+9Cf6+a3BtQ77A+4Mcx+93IaquldVn4gqKgz7Dq6hQR0uyT8Y8bk24IO4Bir7cfvOX+hHU/IYv7HfA6/hWhU+yqEDUeSyl+BKQy6Oim0ZriHBL3DbYiMxGl35E69zcSd5e3AtIM/1329vfB93Vf2KiDT6v5t6ud6dwLdwJR59jh13nNuGa/33OO7kpy/bf6dfxg5c3fWnI34bn8P9FjfhjqN/wDWwSYTf4L7T13EN05biTmI6u5m2AbefvyQiTbj9fjXu+4r7O4vwe9yxbyfuJOPzUeOfwm3nJ4Afq2r4Jvuv4H6bDT7mP9J7v8E1uHsN18r4wYhxIdxJxA7c/ns6PSRU6f73OfhE5B/AH1Q1Kb2aiMhUXF3LPHH3q2xQ1Xj1g72d721+vv258u1ufh8FjlbV7s6yhg0ReRLXfHpY9nIjIi8BN6lqTw05TBKIyGeAS1Q1urFFSvMlHTepanQxs/FS4uZ0ca0Pj6dv2b/ffJ3SmyLyYb98EZFje/NZX/yR418X45rWr01gbHcO94Q3HInI6SJS5osFP45rtv23oOMaLkRkvIicIq4KZxbu6uehoOPqibju/87x+005rlom5eMOUuBJT0RuxxUnfMEXVSVjGXfj6i5micg2EbkCuAy4QkRew90LdEG8eUSYAyzzn/snrpltwpKeGbZm4Ypv9uMOuBeqarzbc0xiZeOqORpwDd/+jCvCT3WCKzrfhyveXIcr5jUxpEzxpjHGGJNsgV/pGWOMMYMl6Y+BCFpxcbFOnTo16DCMMSZtLF++fLeqJuoe35Qy5JPe1KlTWbZsWdBhGGNM2hCR/nQ+khaseNMYY8ywYUnPGGPMsJESSU9EJonIP0VkrYisEZGru5nmDHFP+13p/6xZrjHGmD5JlTq9DuDLqrrC94u3XEQe6+b+t2dU9dwA4jPGGDMEpMSVnn8G0gr/ugF3g2WfHkdijDHG9CQlkl4k30fmcbhe/aOdLCKvichffS/eseZxpYgsE5FltbW1SYrUGGNMukmppOeffvAArkuy+qjRK4Apqnos7ukFf4r+fJiq3qyqi1R1UUnJkLzVxBhjTD+kTNIT9/y3B4C7VPXB6PGqWq+qjf71UiDLd/icFD99rJIVb+1L1uyNMSYpVJWnK2u55Zm+PpJweEiJpCciAvwWWKeqP4kxTZmfDhFZjIt9TzLi2dfUxl0vvcUHf/U8V96xjI27GpOxGGOMSajXt+3nI795ictvfZm7XnqL1o7uHqs3vKVEh9MicirwDO7BqeGn3n4d/1h4Vb1JRK7CPRywAzgAfElVe3wS+aJFi7Q/PbI0tXbwu+fe5NdPbQKBx790OqWjcvs8H2OMGQyVNQ287/pnGJWbxefeOYNLT5xMTmZGv+YlIstVdVGCQ0wJKZH0kqm/SS9sU20jZ/38Gc6cW8ovP3J8AiMzxpjE6OpSLr75Bap2NfL4l06nOD9nQPMbykkvJYo3U9n0knw+944ZPPJ6Nf9cvyvocIwx5gj3Ld/KK5v38fWz5ww44Q11lvR64crTpzNjXD7f+NNqmts6gg7HGGMO2t3Yyg+WrmfxtDF8eNHEoMNJeZb0eiEnM4PvXTCP7fsPcP/ybUGHY4wxB9367Js0tLTzgw/Mw7f1M3FY0uulk48ay9zxo7h32dagQzHGGAA6u5QHVmzjjFnjmDGuIOhw0oIlvT64aNFEVm+vZ+2O6PvmjTFm8D1dVUtNfSsXWbFmr1nS64MLFpSTnRHivuV2tWeMCd59y7YyZmQ275xdGnQoacOSXh+MHpnNmUeX8qdXt9tNn8aYQO1tauOxtTW8f0E52Zl2KO8t21J9dNGiSexrbueJdXb7gjEmOH9euZ32TuWiE6xosy8s6fXRqTOKGV+Yy4MrrBWnMSY4D67YzvzyQmaXjQo6lLRiSa+PMkLCmXNLeW7jHiviNMYEYldDC6u213HWvLKgQ0k7lvT64fSKEg60d7Jssz2FwRgz+J6p3A24Y5HpG0t6/XDyUWPJzgjx5Aar1zPGDL6nKmspzs9h7ngr2uwrS3r9kJedyeJpY3iq0p7KbowZXJ1dyjNVtZxWUUwoZD2w9JUlvX46vaKEyppGduw/EHQoxphh5PVt+9nX3G5Fm/1kSa+fzpjldji72jPGDKanKmsRgdNmWtLrD0t6/TRjXD4TCnN5aoMlPWPM4HmqspZjJxYxemR20KGkJUt6/SQinD6rhOc27qa9s6vnDxhjzADta2rjta37rWhzACzpDcCpM0poaO1gjXVAbYwZBC9v3kuXwqkzi4MOJW1Z0huARVNHA7Bs896AIzHGDAfLt+wjOyPE/PLCoENJW5b0BqB0VC4TR49g+Ra7Sd0Yk3zLNu9l/sRCcrMygg4lbVnSG6BFU0azbMs+VDXoUIwxQ1hLeyert9ezaMrooENJa5b0Bmjh1DHUNrSyda/dr2eMSZ5V2+to6+xioSW9AbGkN0Dhs65lW6xezxiTPOG+fi3pDYwlvQGqKC2gICeTZVavZ4xJouVb9jK9eCRj83OCDiWtWdIboIyQcNyU0Sy3Jy4YY5JEVVm+ZZ9d5SWAJb0EWDRlNJW7Gqg70B50KMaYIeiN2ib2NbcfvE3K9J8lvQRYNGU0qvDqW3a1Z4xJvOW+zcDCKWMCjiT9WdJLgAWTi8gICSusXs8YkwQrtuynKC+Lo0pGBh1K2rOklwB52ZnMKMln1fa6oEMxxgxBr2+vY355ISL2/LyBsqSXIPMnFrJqe53dpG6MSaiW9k4qaxo4ZqJ1PZYIlvQSZH55Ibsb29hZ3xJ0KMaYIWRddT2dXWr9bSZISiQ9EZkkIv8UkbUiskZEru5mGhGR60Vko4i8LiLHBxFrLPP8DrlqmxVxGmMSZ7WvNplnSS8hUiLpAR3Al1V1LnAS8FkRmRs1zdnATP93JXDj4IYY39zxowjJoR3UGGMSYdX2OkbnZVFeNCLoUIaElEh6qlqtqiv86wZgHVAeNdkFwB3qvAgUicj4QQ41phHZGcwcV2CNWYwxCbVqez3zrBFLwqRE0oskIlOB44CXokaVA1sj3m/jyMQYnseVIrJMRJbV1tYmI8xuzSsvZNX2emvMYoxJiJb2TqqsEUtCpVTSE5F84AHgC6ra78eRq+rNqrpIVReVlJQkLsAezC8fxe7GVmrqWwdtmcaYoWv9zgY6rBFLQqVM0hORLFzCu0tVH+xmku3ApIj3E/2wlDHfn429vm1/wJEYY4aCVf5YYo1YEiclkp64wurfAutU9ScxJnsYuNy34jwJqFPV6kELshfmji+0xizGmISxRiyJlxl0AN4pwMeAVSKy0g/7OjAZQFVvApYC5wAbgWbgEwHEGZc1ZjHGJJI1Ykm8lEh6qvosEPdbVdc65LODE1H/zSsv5OmqwWs8Y4wZmlo7XCOWd8yaHnQoQ0pKFG8OJXPGF1Db0MruRmvMYozpv6qaRjq6lLkTRgUdypBiSS/B5o53O+j66oaAIzHGpLP1O90xZM54S3qJZEkvwWb7HXRddb/vuDDGGNZV15ObFWLqWHucUCJZ0kuwMSOzKR2Vw7qdlvSMMf23fmc9s0oLyAhZI5ZEsqSXBLPLRrHOijeNMf2kqqyrbmB2mRVtJpolvSSYM34UG3c10NbRFXQoxpg0VNvQyt6mNuaMLwg6lCHHkl4SzBlfQHunsml3Y9ChGGPS0FrfJmC2NWJJOEt6STDHGrMYYwYgXD0yx4o3E86SXhJMLx5JdkbIblswxvTL+p31lBeNoDAvK+hQhhxLekmQmRFiZmn+wSIKY4zpi3XV9cwus/q8ZLCklyRzxo86eHOpMcb0VmtHJ2/UNtlN6UliSS9JZpdZd2TGmL6rqmmks0uZbS03k8KSXpLMtcYsxph+CJcQ2T16yWFJL0kqfHl8ZY3dtmCM6b3KmgayM0NMK7bux5LBkl6SFOfnMHZkNlU1Vq9njOm9ypoGZpTkW/djSWJJL4lmluazwZKeMaYPKnc2UFGaH3QYQ5YlvSSqKC2gqqYR9/xbY4yJr6GlnR11LcwstUYsyWJJL4kqSgtobO1gR11L0KEYY9JA1S7XBmCWJb2ksaSXRBWl4cYsVsRpjOlZpW+5WWFJL2ks6SVRuFzeGrMYY3qjsqaREVkZTBw9IuhQhixLeklUlJfNuIIcNuy02xaMMT2rrGlgZmk+IWu5mTSW9JKsorSAql12pWeM6VllTQMzx1nRZjJZ0kuycAvOri5rwWmMiW1/cxu7GlqZVWa3KySTJb0kqyjN50B7J9v2HQg6FGNMCgv33mS3KySXJb0km2ktOI0xvRA+RljLzeSypJdk4Rac1jOLMSaeqpoG8nMymVCYG3QoQ5olvSQryM1iQmGu3bZgjIlrg2+5KWItN5PJkt4gmFlawAZ72oIxJo6qmkYqrOVm0lnSGwSzygp4o9Y9GNIYY6LtbmxlT1PbwUeSmeSxpDcIZo7Lp62jiy17moIOxRiTgg41YrHbFZLNkt4gmFVmLTiNMbFV1VhH04MlZZKeiNwqIrtEZHWM8WeISJ2IrPR/3xrsGPtrxjh39mZPUTfGdGdDTQOFI7IoKcgJOpQhLylJT5yPhhOTiEwWkcU9fOw24KwepnlGVRf4v+8mItbBkJedyaQxI+y2BWNMt6pq3INjreVm8iXrSu9XwMnApf59A/DLeB9Q1aeBvUmKJ3CzSgvstgVjzBFUlcqaRrspfZAkK+mdqKqfBVoAVHUfkJ2A+Z4sIq+JyF9F5OhYE4nIlSKyTESW1dbWJmCxAzeztIBNtU20dXQFHYoxJoXsamil7kC7Jb1Bkqyk1y4iGYACiEgJMNCj/QpgiqoeC9wA/CnWhKp6s6ouUtVFJSUlA1xsYswqLaCjS9lsLTiNMRGs+7HBlaykdz3wEDBORP4TeBb4wUBmqKr1qtroXy8FskSkeMCRDpKZpeHGLFbEaYw5JNzAzW5XGByZyZipqt4lIsuBdwECvF9V1w1kniJSBtSoqvpGMSFgz8CjHRxHleQTEqjc2QDHBB2NMSZVVO5sYOzIbMbmW8vNwZDQpCciYyLe7gLujhynqjEbqojI3cAZQLGIbAO+DWQBqOpNwIXAZ0SkAzgAXKKqadPFSW5WBlPHjrTbFowxh6nc1WBFm4Mo0Vd6y3H1eAJMBvb510XAW8C0WB9U1UtjjfPjfwH8ImGRBmBmaT6V9hR1Y4ynqlTVNPKh48uDDmXYSGidnqpOU9XpwOPAeaparKpjgXOBRxO5rHRUUVrA5t1NtLR3Bh2KMSYF7KhrobG1wx4cO4iS1ZDlJN/YBABV/SvwtiQtK21UlBbQpbCp1lpwGmOs5WYQkpX0dojIN0Rkqv+7FtiRpGWljQp7iroxJkLlTncssD43B0+ykt6lQAnutoWHgHEc6p1l2JpWPJLMkFjSM8YA7naF0lE5FOZlBR3KsJGsWxb2AlcnY97pLDszxLTikZb0jDGAK/Wxos3BlZSkJyL/xPfGEklV35mM5aWTirICVm2rCzoMY0zAurqUql0NXHbilKBDGVaSkvSAr0S8zgU+BHQkaVlppWJcAUtXVdPc1kFedrI2vzEm1W3d10xLe5f1xDLIklW8uTxq0HMi8nIylpVuZpXlowobdzVyzMSioMMxxgTkUPdjVrw5mJL1PL0xEX/FIvJeoDAZy0o3Mw+24LSeWYwZzsJ1+3aP3uBKVvlaZM8sHcCbwBVJWlZamTImj+yMkDVmMWaYq6xpoLxoBPk5Vs0xmJK1teeoakvkABGx3lSBzIwQR43Lt6RnzDDnHhxr9XmDLVn36T3fzbAXkrSstFNRmn/wplRjzPDT0dnFG7vsaelBSPRTFsqAcmCEiByHK94EGAXkJXJZ6ayitIA/r9xBQ0s7Bbl2U6oxw82Wvc20dXZZ0gtAoos33wssASYCP4kY3gB8PcHLSlvhHb1qVyPHTx4dcDTGmMEWLumxpDf4Epr0VPV24HYR+ZCqPpDIeQ8l4X72Knc2WNIzZhiqrGlEBGaMszq9wZbo4s2PquqdwFQR+VL0eFX9STcfG3Ymjh7BiKwMu23BmGGqsqaByWPyGJGdEXQow06iizdH+v92+hJHKCTugbLWgtOYYcn63AxOoos3f+3/fyeR8x2KZo4r4Jmq2qDDMMYMsraOLt7c3cR7ji4NOpRhKVkdTpcA/wJMjVyGqn4yGctLR7PK8nlgxTb2N7dRlJcddDjGmEHy5u4mOrrUrvQCkqyb0/8MPAM8DnQmaRlpLbI7ssXTxjW/iS4AABzjSURBVAQcjTFmsGywp6UHKllJL09Vv5qkeQ8JkU9Rt6RnzPBRVdNARkiYXjKy54lNwiWrR5a/iMg5SZr3kDChMJf8nExrzGLMMLNhZwNTx+aRk2ktN4OQrKR3NS7xHRCRehFpEJH6JC0rLYlYC05jhqMq634sUElJeqpaoKohVR2hqqP8+1HJWFY6m1VaYPfqGTOMtLR3smVPkyW9ACWr9ebx3QyuA7aoqj1B3ZtZWsA9r2xld2Mrxfn2EApjhrqNuxrpUmvEEqRkNWT5FXA8sMq/nw+sBgpF5DOq+miSlptWIrsjK55hSc+Yoa5ql6vOmFVm/XcEJVl1ejuA41R1oaouBBYAm4Azgf9O0jLTTvhZWlavZ8zwsGFnI1kZwpSx1nIzKMlKehWquib8RlXXArNVdVOSlpeWSgpyKMrLonKX1esZMxxU1TRwVEk+WRnJOvSaniSreHONiNwI3OPfXwys9U9Pb0/SMtOOiFAxrsAeKGvMMLGhpoHj7MkqgUrW6cYSYCPwBf+3yQ9rB96RpGWmpYqyfDbUNKCqQYdijEmixtYOtu07wKxSq88LUlKu9FT1APC//i+aleVFmF02ijtb3mL7/gNMHG0PlzdmqNqw092qPLvM7t4KUlKu9ERkpojcLyJrRWRT+K+Hz9wqIrtEZHWM8SIi14vIRhF5PcZtEWlnznj3A1hfbUWcxgxl6/xvfM4ES3pBSlbx5u+AG4EOXHHmHcCdPXzmNuCsOOPPBmb6vyv9/NPerDJ328K6auuwxpihbP3OegpyM5lQmBt0KMNaspLeCFV9AhBV3aKq1wHvi/cBVX0a2BtnkguAO9R5ESgSkfEJizgg+TmZTB6Tx3przGLMkLauuoE5ZaMQkaBDGdaSlfRaRSQEVInIVSLyAQb+NPVyYGvE+21+WNqbM77ArvSMGcK6upQNOxuYM956YglaMjuczgM+DywEPgZ8PEnLOoKIXCkiy0RkWW1t6j+dfHbZKN7c08SBNnv0oDFD0fb9B2hs7WD2eKvPC1qyWm++4l82Ap9I0Gy3A5Mi3k/0w7pb/s3AzQCLFi1K+XsB5owfharrmeXYSUVBh2OMSbC1viRnjiW9wCU06YnIw/HGq+r5A5j9w8BVInIPcCJQp6rVA5hfyggXeayrrrekZ8wQtL66AZFDXQ+a4CT6Su9kXL3b3cBLQK9rbEXkbuAMoFhEtgHfBrIAVPUmYClwDu6m92YSdwUZuEmj8xiZnWGNWYwZotZV1zN17EjyspPVCZbprUR/A2W4TqUvBT4CPALcHdkPZyyqemkP4xX4bCKCTDWhkDCrrOBgEYgxZmhZv7OeuXZ/XkpIaEMWVe1U1b+p6seBk3BXZU+KyFWJXM5QNHv8KNZX11t3ZMYMMU2tHWzZ22w9saSIhLfeFJEcEfkg7mb0zwLXAw8lejlDzZyyAupbOthR1xJ0KMaYBHJ968LsMrtdIRUkuiHLHcA8XP3bd1S12y7FzJHCrbrW7ainvGhEwNEYYxJlnbXcTCmJvtL7KK6bsKuB50Wk3v81iIhVWMUxZ/woRGDNDttMxgwla3bUUzgii4mj7WQ2FST0Sk9V7cmI/TQyJ5NpxSNZvaMu6FCMMQm0ZnsdR0+w7sdShSWpFDJvQiFrtlvSM2aoaO/sYt3OBuaVFwYdivEs6aWQeeWj2FHXwp7G1qBDMcYkwMZdjbR1dHG03a6QMizppZB5E9zZoNXrGTM0rPYlN3allzos6aWQo33Ss3o9Y4aGNTvqGZmdwbSxI4MOxXiW9FJIYV4Wk8aMYM12u9IzZihYvb2OuRNGEQpZI5ZUYUkvxcybUGhXesYMAZ1dytrq+oMlOCY1WNJLMfPKC9myp5m6A+1Bh2KMGYA3dzfR3NZp9XkpxpJeigm38lprjVmMSWtrdoQbsVjLzVRiSS/FHH2wBacVcRqTzlZvryMnM8SMEnuGXiqxpJdiSgpyKBuVyyq7Sd2YtLZ6ez2zywrIzLDDbCqxbyMFzZ9YyGtb9wcdhjGmnzq7lNe37eeYiUVBh2KiWNJLQQsmFbF5TzP7mtqCDsUY0w8bdzXS1NbJgkmW9FKNJb0UdJz/oby2za72jElHK7fuA2DBZEt6qcaSXgqaP7EQEVhpRZzGpKWVW/czKjfTemJJQZb0UlBBbhYzx+Vb0jMmTb361n6OnVRkPbGkIEt6Keq4SaN5bet+VDXoUIwxfdDU2kFlTcPBagqTWizppagFk4vY19zOlj3NQYdijOmDVdvr6FKrz0tVlvRSVLjVlxVxGpNewr/ZY+12hZRkSS9FVZQWkJedYUnPmDSz8q39TB6Tx9j8nKBDMd2wpJeiMkLC/PJCXrWkZ0xaWbl1v92fl8Is6aWwBZOLWLejnpb2zqBDMcb0ws66FnbWt1jSS2GW9FLYwsmjaevssn44jUkTr2zeC8DxU0YHHImJxZJeCjth6hgAXn5zb8CRGGN64+U395KXncG8CfY4oVRlSS+FjR6ZzazSAl6ypGdMWnj5zb0snDLanqyQwuybSXGLp41h+ea9dHR2BR2KMSaOfU1tbKhp4MRpY4IOxcRhSS/FLZ42hqa2TtZW25PUjUll4fq8xdPGBhyJiceSXopbPM3q9YxJBy+/uZfszBDHTCwMOhQTR8okPRE5S0Q2iMhGEflaN+OXiEitiKz0f58KIs7BVjoql6lj86xez5gU9/LmvSyYVERuVkbQoZg4UiLpiUgG8EvgbGAucKmIzO1m0j+q6gL/d8ugBhmgxdPG8MrmvXR1WefTxqSixtYOVm+vs/q8NJASSQ9YDGxU1U2q2gbcA1wQcEwpY/G0sexvbqdqV2PQoRhjurF8yz669FB1hEldqZL0yoGtEe+3+WHRPiQir4vI/SIyKdbMRORKEVkmIstqa2sTHeugC589vvTmnoAjMcZ056VNe8gICcdPtpvSU12qJL3e+D9gqqoeAzwG3B5rQlW9WVUXqeqikpKSQQswWSaOHsGkMSN4pmp30KEYY7rx7MbdHDepiJE5mUGHYnqQKklvOxB55TbRDztIVfeoaqt/ewuwcJBiC5yIcNrMEp7fuJu2Drtfz5hUsqexlVXb6zi9Iv1PsIeDVEl6rwAzRWSaiGQDlwAPR04gIuMj3p4PrBvE+AJ3ekUJTW2dLN+yL+hQjDERnt24G1U4zZJeWkiJpKeqHcBVwN9xyexeVV0jIt8VkfP9ZJ8XkTUi8hrweWBJMNEG4+SjxpIZEp6uSv86SmOGkqc21DJmZDbzy+3+vHSQMgXQqroUWBo17FsRr68BrhnsuFJFQW4WC6eM5qkNtXz1rNlBh2OMAbq6lKerdnPqjGJCIQk6HNMLKXGlZ3rntIoS1lbXs6uhJehQjDHA2up6dje2Wn1eGrGkl0bCP6xnKq0VpzGpIFzd8PaK4oAjMb1lSS+NzB0/iuL8bKvXMyZFPLWhlrnjRzGuIDfoUEwvWdJLI6GQcFpFCU9uqKXdHjVkTKD2N7exfMs+Tp9lRZvpxJJemjnr6DLqDrTzwhvWO4sxQXpsbQ0dXcrZ88qCDsX0gSW9NHNaRQkjszP46+rqoEMxZlhbuqqaiaNH2K0KacaSXprJzcrgXXNK+fuaGnuaujEBqTvQzrMbd3PO/PGI2K0K6cSSXho6Z34Ze5va7Bl7xgTk8bU1tHda0WY6sqSXhs6YNY687AweWWVFnMYEYemqaiYU5rJgUlHQoZg+sqSXhnKzMnjH7HH8ffVOOu3BssYMqvqWdp6p2s3ZVrSZlizppan3zR/PnqY2nn/DblQ3ZjA9uqaGts4uzplvRZvpyJJemnrn7HEUjsjinle29jyxMSZh7nn5LaYVj7QHxqYpS3ppKjcrgw8dP5FH1+xkd2Nrzx8wxgxYZU0Dy7bs49LFk6xoM01Z0ktjHzlxEu2dygPLtwUdijHDwt0vv0V2RogPHT8x6FBMP1nSS2MzxhWweOoY7n75LbqsQYsxSdXS3skDy7fx3nlljM3PCToc00+W9NLcR06czOY9zby4ybolMyaZlq6qpr6lg0sXTwo6FDMAlvTS3FnzyijKy+L2FzYHHYoxQ5aqcvsLW5hWPJKTp48NOhwzAJb00lxuVgaXnzyVv6+pYV11fdDhGDMkPV21m9e27udTb59mDVjSnCW9IeCKU6ZRkJPJDf+oCjoUY4YcVeXnj1cyoTCXDy+0os10Z0lvCCjMy2LJKVNZumon63fa1Z4xifRM1W5WvLWfz7xjBtmZdshMd/YNDhFXnDqN/JxMbnhiY9ChGDNkqCo/f6KK8YW5XLTIblMYCizpDRFFedl84pSpPLKqmuVb7OkLxiTC31bvZPmWffzbGUeRk5kRdDgmASzpDSGfPv0oyotG8LUHVtHWYc/aM2Yg6g608+2H1zB3/CguXTw56HBMgljSG0JG5mTy/ffPo2pXIzc99UbQ4RiT1v7rb+vZ3djKf33oGDIz7FA5VNg3OcS8Y/Y4zjt2Ar/4x0Y27moIOhxj0tJLm/bwh5fe4pOnTGP+xMKgwzEJZElvCPrWuXPJz83kyjuWU9fcHnQ4xqSVHfsP8Nk/vMrkMXl86T0VQYdjEsyS3hBUUpDDTR9dyNZ9zfzbH5bT3mn1e8b0RlNrB1fcvozW9k5u+fgi8rIzgw7JJJglvSFq8bQx/PCDx/Dcxj1c+9Aqe8K6MT1oae/k83e/yoad9dzwkeOoKC0IOiSTBHYaM4RduHAib+1t5vonqtjX3M71lxzHiGxrdm1MtP3NbfzLHct4ZfM+vvf+eZwxa1zQIZkksSu9Ie5LZ1Zw3XlzeXxdDZf85kW27GkKOiRjUsq66no+eOPzvLa1jusvPY6PnTQl6JBMElnSGwaWnDKNX390IRtrGnjPT5/m+ieqaGnvDDosYwLV2NrB9/+ylnNveJb9ze3c+akTOf/YCUGHZZJMVFOjrkdEzgJ+DmQAt6jqj6LG5wB3AAuBPcDFqrq5p/kuWrRIly1blviA09DOuha+98haHnm9muL8bC4+YRKXnDCZSWPygg5tWFBVWju6ONDmTjgyMoQMETJC7i8zJNaD/yDYuKuBu156iweWb6OhtYNLTpjMV8+aRVFedtChpQwRWa6qi4KOIxlSIumJSAZQCZwJbANeAS5V1bUR0/wbcIyqflpELgE+oKoX9zRvS3pHev6N3dz67Gb+sb6GLoXpxSN524yxzB1fyPSSkZQXjWDUiCzyczLJCA3eQVhVUYUuVbr8/0Pv3TCNGhf5PiRCKASZoRAZ/nU4oYSTS7ykoqq0dyrtnV20dXTR3tlFS3sXze0dNLV2cqCtk6a2jsP/t3bS3N5Bc2snzW2dNLd1RP33r1s7aW7v7LFBUUggJzODkTmZ5Oe4/+515qFh2YeG5eVkuHHZh17nZR8aNyIrw2+L3n2Pqkpnl9Lp/3d0KZ2dh95H/nV0ue8l/D68/TPEbeeMkBAS/HD3HYTk0HcSivheQuKSfm/j7I3mtg72NLbx1t5mNtU2smp7Hc+/sYdt+w6QlSGcNW88nzp1GsdOKkrYMocKS3rJDkLkZOA6VX2vf38NgKr+MGKav/tpXhCRTGAnUKI9rIAlvdi27z/AX1dV89zG3bz85l6a2o4s8szPyaQgNzNu7/LaiwTV1dV9QlMOf59sIriDa8QVliq0dXTR1s9bO7IzQ4zMziAvO5O87Az/51/nZJKXlUFezqHhI7IyCAkuoYQTTKcefN/S7pJqY2snTa0dNLZ20OT/wsMO9LF4Orze7ooy5JJRSA5LYOHXQQufpIQTaMhfBWdEfG/h/+7qGDq73AlLW2f4ZKWTlvbDv8/CEVmcNH0Mp8wo5ux54ykpyAloDVPfUE56qdJ6sxzYGvF+G3BirGlUtUNE6oCxwO7omYnIlcCVAJMnW595sZQXjeBTb5/Op94+na4uZUfdAd6obaKmroX6lnYaWjpoaOmgvqWdjh4SQsgfiMJn9hLxOiQg4g5OPU0TOjjdofeHpo/4vL9qEw6NCyfXw69C3PAOf7XS1XXkVUtIhKxMIScjRFZGiOxM95eVESInM8TInMzDE1m2uwIbkZ1BXlZGIF1UdXYpzW3uCrTJX0k2tnbQ3Nbh/7vk2NLeSWcXdHZ1HUxs4f9dqgcTR0YodPBK61ByPLzoNTwuJEJmRnTikSNOeroirgTD46KvDsPfUWdXl/sf8R11RcUa/n/4d+mWk5khZIW/vwwhJyuD0XnZjBmZxcTReRxVkk/pqBwrPjYpk/QSSlVvBm4Gd6UXcDhpIRQSJo7OY+Joq99LBxkhoSA3i4LcrKBDMSatpErrze1A5COJJ/ph3U7jizcLcQ1ajDHGmF5JlaT3CjBTRKaJSDZwCfBw1DQPAx/3ry8E/tFTfZ4xxhgTKSWKN30d3VXA33G3LNyqqmtE5LvAMlV9GPgt8HsR2QjsxSVGY4wxptdSIukBqOpSYGnUsG9FvG4BPjzYcRljjBk6UqV40xhjjEk6S3rGGGOGDUt6xhhjhg1LesYYY4aNlOiGLJlEpBbY0s+PF9NNjy9pxOIPVjrHn86xg8U/UFNUtSTA5SfNkE96AyEiy9K5/zmLP1jpHH86xw4Wv4nNijeNMcYMG5b0jDHGDBuW9OK7OegABsjiD1Y6x5/OsYPFb2KwOj1jjDHDhl3pGWOMGTYs6RljjBk2LOl1Q0TOEpENIrJRRL4WdDw9EZFJIvJPEVkrImtE5Go/fIyIPCYiVf7/6KBjjUdEMkTkVRH5i38/TURe8t/DH/1jp1KSiBSJyP0isl5E1onIyem0/UXki37fWS0id4tIbipvfxG5VUR2icjqiGHdbm9xrvfr8bqIHB9c5Adj7S7+//H7z+si8pCIFEWMu8bHv0FE3htM1EODJb0oIpIB/BI4G5gLXCoic4ONqkcdwJdVdS5wEvBZH/PXgCdUdSbwhH+fyq4G1kW8/y/gp6o6A9gHXBFIVL3zc+BvqjobOBa3Hmmx/UWkHPg8sEhV5+Ee73UJqb39bwPOihoWa3ufDcz0f1cCNw5SjPHcxpHxPwbMU9VjgErgGgD/W74EONp/5lf+OGX6wZLekRYDG1V1k6q2AfcAFwQcU1yqWq2qK/zrBtwBtxwX9+1+stuB9wcTYc9EZCLwPuAW/16AdwL3+0lSNn4RKQROwz3zEVVtU9X9pNH2xz1mbISIZAJ5QDUpvP1V9WncczUjxdreFwB3qPMiUCQi4wcn0u51F7+qPqqqHf7ti8BE//oC4B5VbVXVN4GNuOOU6QdLekcqB7ZGvN/mh6UFEZkKHAe8BJSqarUftRMoDSis3vgZ8B9Al38/FtgfcRBI5e9hGlAL/M4Xz94iIiNJk+2vqtuBHwNv4ZJdHbCc9Nn+YbG2dzr+pj8J/NW/Tsf4U5YlvSFERPKBB4AvqGp95Dh196ak5P0pInIusEtVlwcdSz9lAscDN6rqcUATUUWZKb79R+OuJqYBE4CRHFn0llZSeXv3RESuxVVZ3BV0LEORJb0jbQcmRbyf6IelNBHJwiW8u1T1QT+4JlyM4//vCiq+HpwCnC8im3HFye/E1ZEV+eI2SO3vYRuwTVVf8u/vxyXBdNn+7wbeVNVaVW0HHsR9J+my/cNibe+0+U2LyBLgXOAyPXQTddrEnw4s6R3pFWCmb7mWjatAfjjgmOLy9V+/Bdap6k8iRj0MfNy//jjw58GOrTdU9RpVnaiqU3Hb+x+qehnwT+BCP1kqx78T2Cois/ygdwFrSZPtjyvWPElE8vy+FI4/LbZ/hFjb+2Hgct+K8ySgLqIYNGWIyFm4Iv7zVbU5YtTDwCUikiMi03ANcl4OIsYhQVXtL+oPOAfXeuoN4Nqg4+lFvKfiinJeB1b6v3Nw9WJPAFXA48CYoGPtxbqcAfzFv56O+3FvBO4DcoKOL07cC4Bl/jv4EzA6nbY/8B1gPbAa+D2Qk8rbH7gbV//YjrvSviLW9gYE1yL7DWAVrpVqKsa/EVd3F/4N3xQx/bU+/g3A2UHHn85/1g2ZMcaYYcOKN40xxgwblvSMMcYMG5b0jDHGDBuW9IwxxgwblvSMMcYMG5b0zKATkU4RWRnxNzXomBJFRI4Tkd/612eISF3Een4rYrojetn3w2P2tB8xzQQRuT96eJyYNovIAxHvLxSR2/q1gvGXc0u8ztlFZImITIh4f4+IzEx0HMbEY0nPBOGAqi6I+NscHuFvIE7n/fLrwPUR75+JWM/vRgy/je67+uq2p/1IqrpDVS884pPxLUz200JU9VOqujbOJEtw3ZyF3Yi7GduYQZPOBxczRIjIVP+csDtwN0dPEpF/F5FX/BXPdyKmvVZEKkXkWXHPffuKH/6kiCzyr4t9l2bhZ/T9T8S8/tUPP8N/JvwMvLt8bySIyAki8ryIvCYiL4tIgYg8LSILIuJ4VkSOjVqPAuAYVX2tp3XW7p8SgMbuaT96e632r4/2Ma706xfryul/cTc4R84nJO7ZcyUR7zeKSInvkegFEVklIt8XkcaI7faXiHn8wneddfA78Nv8NnHP5lsl7ll9FwKLgLt8rCOAZ4B3y6GuzoxJOkt6JggjIor8HvLDZgK/UtWjgVn+/WJcTycLReQ0EVmI66ZsAa7HmRN6sawrcN1OneCn/xfflRO4p1F8AffcxOnAKeK6nvsjcLWqHovrl/IArpu3JQAiUgHkdpPcFuGSdqSTffL8q4gc3Yt4I0X2tB/Lp4Gfq+oCv/xtMaa7FzheRGaEB6hqF3AncJkf9G7gNVWtxfV9eqOqzsf1HNIXC4ByVZ3nP/87Vb0f12PNZf6q94Bf/kbc8weNGRSW9EwQIos3P+CHbVH3rDOA9/i/V4EVwGxcEnw78JCqNqt7ikRv+kR9D67fxZW4xy2N9fMCeFlVt/mD70pgKi7hVqvqKwCqWu+vvO4DzhXXsfcnccWT0cbjHjEUtgKY4pPnDbjuyXpFet/T/gvA10Xkq35ZB2JM1wn8D0cWl94KXO5ffxL4nX99Cq6rLHDdkvXFJmC6iNwgrj/J+jjT7uLwIk9jksqSnkkVTRGvBfhhRGKcoaq/7eHzHRzan3Oj5vW5iHlNU9VH/bjWiOk6cY8I6pa6DoAfwz2C5yK6T0YHIpftE2ajf70UyBKR4h7WI1ZP+7Hi+gNwvl/2UhF5Z5zJf4972O3BHvtVdSvu6QTvxF1ZR15ZdrfsyO0Mh2/r8Dz34a7ensRdid4SJ6ZcH7sxg8KSnklFfwc+Ke75gIhIuYiMA54G3i8iI3z92XkRn9kMLPSvL4ya12f8FRoiUiHuAa+xbADGi8gJfvqCiDqnW3CNVF7xB/Zo64CDxYciUhZRT7gY93vbE2/FJXZP+7Gmnw5sUtXrcU8VOCbWtOoeG/RT4ItRo27BFXPep6qdfthzuKJkOFT8CbAFmCuux/8i3BMZomMqBkKq+gDwDdxjlgAagIKoySs4skjYmKSxpGdSjr8S+wPwgoiswj2frkBVV+Dq217DXZG8EvGxH+OS26tA5NXULbjH5KzwjT9+TfwrujbgYuAGEXkNd3WX68ctxxXV/S7GZ9cDhT4hg0u+q/18rgcuCV+5icjduKLJWSKyTUSu8J/5BS4xPObrPG+Kv7W4yC9jJTAPuKOH6X/Lkev/MJAftV5XA5/12//gU7r9leG9uER1L64IOlo58KSP6U4OFaneBtwUbsgiIqW4ou6dPcRsTMLYUxZM2hKR64BGVf3xIC1vAq7IbravB+xumi8CDaoar0gvpfhWrz9V1bfHmaZRVfMTvNwvAvW9KLo2JmHsSs+YXhCRy3ENYa6NlfC8Gzm8rjClicjXgAfo5n7AQbAfuD2A5ZphzK70jDHGDBt2pWeMMWbYsKRnjDFm2LCkZ4wxZtiwpGeMMWbYsKRnjDFm2Ph/2hPcK/6KS8EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "N = 4\n", + "f, H_im = sig.freqz(h)\n", + "posfreq = np.square(H[0:512//N])\n", + "negfreq = np.flipud(np.square(H[0:512//N]))\n", + "plt.plot((np.abs(posfreq) + np.abs(negfreq)))\n", + "plt.xlabel('Frequency (512 is Nyquist)')\n", + "plt.ylabel('Magnitude')\n", + "plt.title('Unity Condition, Sum of Squared Magnitude of 2 Neighboring Subbands')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "b = sig.firwin(80, 0.5, window=('kaiser', 8))" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb0AAAEWCAYAAADy9UlpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nO3deZwcZZnA8d/Tc2Yyk5kkM5lJJjfJ5CCBQEIAQcADBeTwQA5RjOKyuqJ47Srigcequ+t6gAoiIiAIciorUTmU+0xCIPdMCAk5JpPJNWfmfvaP9+2k05nuubqnumee7+czn+muqq56qrq6nqr3festUVWMMcaY4SAUdADGGGPMYLGkZ4wxZtiwpGeMMWbYsKRnjDFm2LCkZ4wxZtiwpGeMMWbYSHrSE5GbROSbyV5OMoiIisgM/zrueojI10XklsGLLrWIyAgR+T8RqROR+4KOJ5FEZLOIvDvoOGJJ9G9MRG4Tke8nan4R8y0VkadFpEFE/jfR8+9HPGtE5IxeThtzHxCRM0RkWwLjukxEHk3U/BJFRJ4UkU/FGDfVHy8zByGOAS2rx6QXeeCPGHadiNzZmwWo6qdV9Xv+cwPeOUSkQkTuE5Hd/gD7uoh8SUQyBjLfnvS0Hqr6A1XtdodIJBGZKCIPRKz/ahFZkuzl9sKFQCkwVlU/HD1SRIpE5FYR2ekPepUi8rXBDzOxfIJQEbkgavhP/fAlyY4h0b+xJLoS2A2MUtUvR48UkX/3+3ODiLwpIv8ea0YRB76lUcPvFJHrehOMqh6tqk/2bRWST1XvUtX3JGPeInKqiDzvjx17ReQ5ETkhGctKVWlVvCkiRwEvAVuB+apaCHwYWAQUBBnbIPo9bv2nAGOBjwE1gUbkTAEqVbUjxvifAvnAHKAQOB/YOEixHZSkM9FK4PKoZVwEvJGEZaWzKcBajd0jhuC242jgLOAqEbmkh3meKCJvS2CMgUrmlZKIjAL+AtwAjAHKge8ArclaZkpS1bh/gAIzooZdB9zpX58BbAO+DOwCqoFPREx7G/B9YCRwAOgCGv3fBKAZd3UQnv54oBbI6iaWO4FHeoj3fGANsB94EpgTMW4z8BXgdaAO+COQGzH+3338O4BPRq57L9bj4DYZaBw9rF8jsCDGuDOAbVHDNgPvjvje7vPbsQFYBVQA1/jvbivwnjjLnuPXZb9ft/P98O8AbUC7j++Kbj67Gnh/nHmfCaz32+MXwFPAp6L3N/9+qv9uMv37TwDr/DptAv41epsAXwV24k4aQsDXcElpD3AvMCbiMx8Dtvhx10Zuw27ivg34Me7EY7Qfdi7wV+BZYIkfdhTwDz/P3cBdQFHUfv+qX4f7/D7x/QT9xm4Lz6u7/QQ4Dljhl/1H4J6o6c8FVvrv/XngmDjf49uAV/z3+ArwtogY2/1+0hhre0bN63rghhjjwvvAV4F/Rh0jrutN7Bz+2xgB3A7s8/vSf0Rto83E+M1GfD9f99/tZuCyiM8WAnfgjmtbgG8AIT9uCfAc7qRwj/8elwDPRh2DPw1U+fX4JSB+XAbwv365bwJXEfHbiNpmi4D9cbb3dcT/nT0J/BB4GagH/oz/3URMeyXu+FkNfCViXouBF3z81bjfeHYf1vHHfh03AZ+NimuJH97gt8FlsdZRVRN2pVeG+2LLgSuAX4rI6MgJVLUJOBvYoar5/m8HbkNeFDHpx4B7VLW9m+W8G7g/VhAiUgHcDXwBKAGWAv8nItkRk12EO4ucBhyD22CIyFm4nfpMYKZf1hHirEdC4uiFF3Hb9xIRmdzLz0Q6D3fgH407yP4dlwTKge8Cv+7uQyKSBfwf8CgwDvgccJeIzFLVbwM/AP7ot8dvY8T9nyLyCRGZGTXvYuBB3MGgGJeMTunDOu3CHdxG4RLgT0Xk+IjxZbgz2ym4H+XngPcDp+OSwj7cjwwRmQvciNsPJ+Cupif2sPwW3AEgfFVyOe4gd9hq4g4YE3AnD5NwBxn8fvEQLjGMwe07H4j6/EB+YzH5Zf8Jt0+MwSXcD0WMPw64FfhX3Lb4NfCwiOR0M68xwCO4ZDUW+AnwiIiMVdUluET/3z6ux3uIS4C3406u4vkVUNFdfVtfYge+jTtwT8cdAz7azTTxfrNluH23HPg4cLOIzPLjbsB9d9Nx+9zluP007ETcQbsU+M8Y63kucIJf7kXAe/3wf8F95wtwJ07vj/F5cCUSnSJyu4icHb3/9NLluAuC8UAH7ruO9A7c8fM9wFcjvpdO4Iu4bXQy8C7g36I+G28dz8WdnC3CVaUAICIjfQxnq2oB7qRrZbwVSFTSawe+q6rtqroUdyY3q4fPhN2O38F8vdyluB9gd8bizhJiuRh3JfiYT5o/xp3B/UlEduEOONer6g5V3Ys7iC/wn70I+J2qrvYHj+viLOcJYJyIrBSRh/sQR2QxTKw4evJh4Bngm8CbPoa+lMk/o6p/V1cMeR8uKf/Ix3kPMFVEirr53Em44skfqWqbqv4DV1RyaS+X+zncQe8qYK2IbBSRs/24c4A1qnq/j+NnuKuyXlHVR1T1DXWewiXmt0dM0gV8W1VbVfUA7ozyWlXdpqqtuO/6Ql+0dCHwF1V92o/7pv98T+4ALvfb7nRcIomMcaPfH1pVtRaXEE73o08CMnH7RLuqPog7m440kN9YPCcBWcDP/Lzvx12hhV0J/FpVX1LVTlW9HVccdlI383ofUKWqv1fVDlW9G3f1fl4/4roOd3z6XQ/THcAliu4a3vQl9ouAH6jqPlXdxpEHc+j5N/tN//0+hUv+F/lj2iXANaraoKqbcVdmH4v43A5VvcFvswMx1vNHqrpfVd8C/snhx62f+315H/CjGJ9HVeuBU3FXSb8BakXkYREpjfWZbvw+4hj5zYh1DPuOqjap6ircd3epX/ZyVX3Rr+Nm3AnI6VHzjreOP1PVrX7b/zDqc13APBEZoarVqhr3RKk3Sa8T96OIlIX7EYbt0cPrcppxB8je+DMwV0Sm4c6w6lQ1+gd/cDm4M4xYJuCKDwBQ1S5ckd0y3BkaHH4wjYxzgp82bAuxtQG7VHWBqp7fhzjKI6aJFUdc/kf5NVU9GndmuBKX1KU3n+fw+r8DwG5V7Yx4T4xYJgBb/bqEbeHwdYoX9wF1jX0W4k5e7gXu81cHh217dWUWW7uf05H8WeuLvmJ+Py6JFkdMUquqLRHvpwAPich+P/063H5e2k0sTbj9rqf1exZ3AnEtLmkedvAS13LxHhHZLiL1uGK4cIwTgO1+vcOi138gv7F4ult25L4/BfhyeFv57TXJf667eUX/bnq9j4SJyFW4K4r3+ROPntwClIpIdHLta+yR27y7/S/eb3af31fCtvh5FuOOl1uixkVuk97s6709bsWdl6quU9UlqjoRmOc//7NeLL+7+W/BrVtxnPET4GADxL/4hmz1uJKhyM9BP47NfptfjDuRrRaRR0RkdrwV6E3Sewt32R9pGvGTQixHVGD7g9G9uKu9jxH7Kg/gcSKKXrqxA7ejAweLSCbh6lL2Rk7oG8V8FDhbRJ7BbeRJEZPEKzrs6dEUseLY3sPn+kRVd+OuIifgiqaagLyI5WbgDsSJsAOYJCKR+8xk+rFO/ozzB7g6qGm4q/eD2z5ie4Udtl64oqTwtDnAA7jtUKqqRbji5MiTgOjvayuuOKQo4i9XVbd3E0seLkn3xp24erfook1w66u4BlijcPteOMZqoDzqxGUS/dPdvhlz+8VYduS+vxX4z6htleev4qIdtt9HzKvX+4iIfBJX3/ouf8XVI1Vtw9Urf4/Dv/e+xF7N4cXYfd3+o31RW9hk3PbYjbtAmBI1LnKbDORRN/2OW1XX44rU5/lB8faT7uY/Gbduu+OMDxev34i76p/p9/+vc/h3Fc9hv0mijs2+5OpM3AXRetxVbEy9SXp/BL4hrql8yJfRnkecurU4aoCxIlIYNfwOXPn4+cRPet8G3iYi/yMiZQAiMsM3Uy7CJc/3ici7fB3Ul3HFGc93M6+bcQfHv+Lq8qYDS0Rkrj/QfTtOHNnABBF5RUS6K0PvSxxHENcU+4wY4/5LROaJSKaIFACfATaq6h5cmX2uiLzPL/cbQHf1F/3xEu7E4D9EJMvHdx6uSLRHIvJNETlBRLJFJBe4GldhvQFXFHS0iHzQFzF+nsN/cCuB00Rkst93rokYl41bx1qgwxeZ9tTc+yZc/eIUH1uJHLrl4H7gXHFNu7Nx9Zy9rQa4Hlda8XQ34wpwRZJ1IlKOazQV9gLuSvMq/71egKv474/ufmMrgXNEZIz/3XwhatkdwOf99/rBqGX/Bvi0iJwozki/f3XXWnoprn7tI349Lgbm4orBeyQil+FODs5U1U29XWHv90Auh0p0+hr7vcA1IjLafz9X9XH5AN/x+/fbcXVQ9/lSlHtx+1uB3+e+hDtBSoR7gatFpNwfA78aa0IRmS0iXxaRif79JFzx44t+kni/s7CPRhwjvwvcH1FSBPBNEckTkaNx9ZZ/9MMLcI1fGv2V2Gf6uI6f9zloNO6kKLxOpSJygT/haMX9xuJWR/Tmx/xd3MH6WVyF/3/jWses7kPQwMEzi7uBTb64YYIf/pwPdIWqxryCVNU3cJWgU4E1IlKHO8tfBjSo6gbcGfQNuLOP84Dz/JngQSKSj6tfuwhXFPZr3IHzZ7iWVHW4FlkAS0VkNe5gFjYFd4Y0A3hQRE6OirNXcXTH74jhlpXdycM1etiPq/yegjtZQFXrcJXDt+DOJJtwrcoGzMd+Hq7SfDeuAcHl/jvt1SxwZfy7cWd/Z+KKrxr9FeuHcfURe3AV4c9FLPsx3I/ndWA5EQdRVW3AJcl7cfvnR4Du6lkj/dxP86iINOB+9Cf6+a3BtQ77A+4Mcx+93IaquldVn4gqKgz7Dq6hQR0uyT8Y8bk24IO4Bir7cfvOX+hHU/IYv7HfA6/hWhU+yqEDUeSyl+BKQy6Oim0ZriHBL3DbYiMxGl35E69zcSd5e3AtIM/1329vfB93Vf2KiDT6v5t6ud6dwLdwJR59jh13nNuGa/33OO7kpy/bf6dfxg5c3fWnI34bn8P9FjfhjqN/wDWwSYTf4L7T13EN05biTmI6u5m2AbefvyQiTbj9fjXu+4r7O4vwe9yxbyfuJOPzUeOfwm3nJ4Afq2r4Jvuv4H6bDT7mP9J7v8E1uHsN18r4wYhxIdxJxA7c/ns6PSRU6f73OfhE5B/AH1Q1Kb2aiMhUXF3LPHH3q2xQ1Xj1g72d721+vv258u1ufh8FjlbV7s6yhg0ReRLXfHpY9nIjIi8BN6lqTw05TBKIyGeAS1Q1urFFSvMlHTepanQxs/FS4uZ0ca0Pj6dv2b/ffJ3SmyLyYb98EZFje/NZX/yR418X45rWr01gbHcO94Q3HInI6SJS5osFP45rtv23oOMaLkRkvIicIq4KZxbu6uehoOPqibju/87x+005rlom5eMOUuBJT0RuxxUnfMEXVSVjGXfj6i5micg2EbkCuAy4QkRew90LdEG8eUSYAyzzn/snrpltwpKeGbZm4Ypv9uMOuBeqarzbc0xiZeOqORpwDd/+jCvCT3WCKzrfhyveXIcr5jUxpEzxpjHGGJNsgV/pGWOMMYMl6Y+BCFpxcbFOnTo16DCMMSZtLF++fLeqJuoe35Qy5JPe1KlTWbZsWdBhGGNM2hCR/nQ+khaseNMYY8ywYUnPGGPMsJESSU9EJonIP0VkrYisEZGru5nmDHFP+13p/6xZrjHGmD5JlTq9DuDLqrrC94u3XEQe6+b+t2dU9dwA4jPGGDMEpMSVnn8G0gr/ugF3g2WfHkdijDHG9CQlkl4k30fmcbhe/aOdLCKvichffS/eseZxpYgsE5FltbW1SYrUGGNMukmppOeffvAArkuy+qjRK4Apqnos7ukFf4r+fJiq3qyqi1R1UUnJkLzVxBhjTD+kTNIT9/y3B4C7VPXB6PGqWq+qjf71UiDLd/icFD99rJIVb+1L1uyNMSYpVJWnK2u55Zm+PpJweEiJpCciAvwWWKeqP4kxTZmfDhFZjIt9TzLi2dfUxl0vvcUHf/U8V96xjI27GpOxGGOMSajXt+3nI795ictvfZm7XnqL1o7uHqs3vKVEh9MicirwDO7BqeGn3n4d/1h4Vb1JRK7CPRywAzgAfElVe3wS+aJFi7Q/PbI0tXbwu+fe5NdPbQKBx790OqWjcvs8H2OMGQyVNQ287/pnGJWbxefeOYNLT5xMTmZGv+YlIstVdVGCQ0wJKZH0kqm/SS9sU20jZ/38Gc6cW8ovP3J8AiMzxpjE6OpSLr75Bap2NfL4l06nOD9nQPMbykkvJYo3U9n0knw+944ZPPJ6Nf9cvyvocIwx5gj3Ld/KK5v38fWz5ww44Q11lvR64crTpzNjXD7f+NNqmts6gg7HGGMO2t3Yyg+WrmfxtDF8eNHEoMNJeZb0eiEnM4PvXTCP7fsPcP/ybUGHY4wxB9367Js0tLTzgw/Mw7f1M3FY0uulk48ay9zxo7h32dagQzHGGAA6u5QHVmzjjFnjmDGuIOhw0oIlvT64aNFEVm+vZ+2O6PvmjTFm8D1dVUtNfSsXWbFmr1nS64MLFpSTnRHivuV2tWeMCd59y7YyZmQ275xdGnQoacOSXh+MHpnNmUeX8qdXt9tNn8aYQO1tauOxtTW8f0E52Zl2KO8t21J9dNGiSexrbueJdXb7gjEmOH9euZ32TuWiE6xosy8s6fXRqTOKGV+Yy4MrrBWnMSY4D67YzvzyQmaXjQo6lLRiSa+PMkLCmXNLeW7jHiviNMYEYldDC6u213HWvLKgQ0k7lvT64fSKEg60d7Jssz2FwRgz+J6p3A24Y5HpG0t6/XDyUWPJzgjx5Aar1zPGDL6nKmspzs9h7ngr2uwrS3r9kJedyeJpY3iq0p7KbowZXJ1dyjNVtZxWUUwoZD2w9JUlvX46vaKEyppGduw/EHQoxphh5PVt+9nX3G5Fm/1kSa+fzpjldji72jPGDKanKmsRgdNmWtLrD0t6/TRjXD4TCnN5aoMlPWPM4HmqspZjJxYxemR20KGkJUt6/SQinD6rhOc27qa9s6vnDxhjzADta2rjta37rWhzACzpDcCpM0poaO1gjXVAbYwZBC9v3kuXwqkzi4MOJW1Z0huARVNHA7Bs896AIzHGDAfLt+wjOyPE/PLCoENJW5b0BqB0VC4TR49g+Ra7Sd0Yk3zLNu9l/sRCcrMygg4lbVnSG6BFU0azbMs+VDXoUIwxQ1hLeyert9ezaMrooENJa5b0Bmjh1DHUNrSyda/dr2eMSZ5V2+to6+xioSW9AbGkN0Dhs65lW6xezxiTPOG+fi3pDYwlvQGqKC2gICeTZVavZ4xJouVb9jK9eCRj83OCDiWtWdIboIyQcNyU0Sy3Jy4YY5JEVVm+ZZ9d5SWAJb0EWDRlNJW7Gqg70B50KMaYIeiN2ib2NbcfvE3K9J8lvQRYNGU0qvDqW3a1Z4xJvOW+zcDCKWMCjiT9WdJLgAWTi8gICSusXs8YkwQrtuynKC+Lo0pGBh1K2rOklwB52ZnMKMln1fa6oEMxxgxBr2+vY355ISL2/LyBsqSXIPMnFrJqe53dpG6MSaiW9k4qaxo4ZqJ1PZYIlvQSZH55Ibsb29hZ3xJ0KMaYIWRddT2dXWr9bSZISiQ9EZkkIv8UkbUiskZEru5mGhGR60Vko4i8LiLHBxFrLPP8DrlqmxVxGmMSZ7WvNplnSS8hUiLpAR3Al1V1LnAS8FkRmRs1zdnATP93JXDj4IYY39zxowjJoR3UGGMSYdX2OkbnZVFeNCLoUIaElEh6qlqtqiv86wZgHVAeNdkFwB3qvAgUicj4QQ41phHZGcwcV2CNWYwxCbVqez3zrBFLwqRE0oskIlOB44CXokaVA1sj3m/jyMQYnseVIrJMRJbV1tYmI8xuzSsvZNX2emvMYoxJiJb2TqqsEUtCpVTSE5F84AHgC6ra78eRq+rNqrpIVReVlJQkLsAezC8fxe7GVmrqWwdtmcaYoWv9zgY6rBFLQqVM0hORLFzCu0tVH+xmku3ApIj3E/2wlDHfn429vm1/wJEYY4aCVf5YYo1YEiclkp64wurfAutU9ScxJnsYuNy34jwJqFPV6kELshfmji+0xizGmISxRiyJlxl0AN4pwMeAVSKy0g/7OjAZQFVvApYC5wAbgWbgEwHEGZc1ZjHGJJI1Ykm8lEh6qvosEPdbVdc65LODE1H/zSsv5OmqwWs8Y4wZmlo7XCOWd8yaHnQoQ0pKFG8OJXPGF1Db0MruRmvMYozpv6qaRjq6lLkTRgUdypBiSS/B5o53O+j66oaAIzHGpLP1O90xZM54S3qJZEkvwWb7HXRddb/vuDDGGNZV15ObFWLqWHucUCJZ0kuwMSOzKR2Vw7qdlvSMMf23fmc9s0oLyAhZI5ZEsqSXBLPLRrHOijeNMf2kqqyrbmB2mRVtJpolvSSYM34UG3c10NbRFXQoxpg0VNvQyt6mNuaMLwg6lCHHkl4SzBlfQHunsml3Y9ChGGPS0FrfJmC2NWJJOEt6STDHGrMYYwYgXD0yx4o3E86SXhJMLx5JdkbIblswxvTL+p31lBeNoDAvK+hQhhxLekmQmRFiZmn+wSIKY4zpi3XV9cwus/q8ZLCklyRzxo86eHOpMcb0VmtHJ2/UNtlN6UliSS9JZpdZd2TGmL6rqmmks0uZbS03k8KSXpLMtcYsxph+CJcQ2T16yWFJL0kqfHl8ZY3dtmCM6b3KmgayM0NMK7bux5LBkl6SFOfnMHZkNlU1Vq9njOm9ypoGZpTkW/djSWJJL4lmluazwZKeMaYPKnc2UFGaH3QYQ5YlvSSqKC2gqqYR9/xbY4yJr6GlnR11LcwstUYsyWJJL4kqSgtobO1gR11L0KEYY9JA1S7XBmCWJb2ksaSXRBWl4cYsVsRpjOlZpW+5WWFJL2ks6SVRuFzeGrMYY3qjsqaREVkZTBw9IuhQhixLeklUlJfNuIIcNuy02xaMMT2rrGlgZmk+IWu5mTSW9JKsorSAql12pWeM6VllTQMzx1nRZjJZ0kuycAvOri5rwWmMiW1/cxu7GlqZVWa3KySTJb0kqyjN50B7J9v2HQg6FGNMCgv33mS3KySXJb0km2ktOI0xvRA+RljLzeSypJdk4Rac1jOLMSaeqpoG8nMymVCYG3QoQ5olvSQryM1iQmGu3bZgjIlrg2+5KWItN5PJkt4gmFlawAZ72oIxJo6qmkYqrOVm0lnSGwSzygp4o9Y9GNIYY6LtbmxlT1PbwUeSmeSxpDcIZo7Lp62jiy17moIOxRiTgg41YrHbFZLNkt4gmFVmLTiNMbFV1VhH04MlZZKeiNwqIrtEZHWM8WeISJ2IrPR/3xrsGPtrxjh39mZPUTfGdGdDTQOFI7IoKcgJOpQhLylJT5yPhhOTiEwWkcU9fOw24KwepnlGVRf4v+8mItbBkJedyaQxI+y2BWNMt6pq3INjreVm8iXrSu9XwMnApf59A/DLeB9Q1aeBvUmKJ3CzSgvstgVjzBFUlcqaRrspfZAkK+mdqKqfBVoAVHUfkJ2A+Z4sIq+JyF9F5OhYE4nIlSKyTESW1dbWJmCxAzeztIBNtU20dXQFHYoxJoXsamil7kC7Jb1Bkqyk1y4iGYACiEgJMNCj/QpgiqoeC9wA/CnWhKp6s6ouUtVFJSUlA1xsYswqLaCjS9lsLTiNMRGs+7HBlaykdz3wEDBORP4TeBb4wUBmqKr1qtroXy8FskSkeMCRDpKZpeHGLFbEaYw5JNzAzW5XGByZyZipqt4lIsuBdwECvF9V1w1kniJSBtSoqvpGMSFgz8CjHRxHleQTEqjc2QDHBB2NMSZVVO5sYOzIbMbmW8vNwZDQpCciYyLe7gLujhynqjEbqojI3cAZQLGIbAO+DWQBqOpNwIXAZ0SkAzgAXKKqadPFSW5WBlPHjrTbFowxh6nc1WBFm4Mo0Vd6y3H1eAJMBvb510XAW8C0WB9U1UtjjfPjfwH8ImGRBmBmaT6V9hR1Y4ynqlTVNPKh48uDDmXYSGidnqpOU9XpwOPAeaparKpjgXOBRxO5rHRUUVrA5t1NtLR3Bh2KMSYF7KhrobG1wx4cO4iS1ZDlJN/YBABV/SvwtiQtK21UlBbQpbCp1lpwGmOs5WYQkpX0dojIN0Rkqv+7FtiRpGWljQp7iroxJkLlTncssD43B0+ykt6lQAnutoWHgHEc6p1l2JpWPJLMkFjSM8YA7naF0lE5FOZlBR3KsJGsWxb2AlcnY97pLDszxLTikZb0jDGAK/Wxos3BlZSkJyL/xPfGEklV35mM5aWTirICVm2rCzoMY0zAurqUql0NXHbilKBDGVaSkvSAr0S8zgU+BHQkaVlppWJcAUtXVdPc1kFedrI2vzEm1W3d10xLe5f1xDLIklW8uTxq0HMi8nIylpVuZpXlowobdzVyzMSioMMxxgTkUPdjVrw5mJL1PL0xEX/FIvJeoDAZy0o3Mw+24LSeWYwZzsJ1+3aP3uBKVvlaZM8sHcCbwBVJWlZamTImj+yMkDVmMWaYq6xpoLxoBPk5Vs0xmJK1teeoakvkABGx3lSBzIwQR43Lt6RnzDDnHhxr9XmDLVn36T3fzbAXkrSstFNRmn/wplRjzPDT0dnFG7vsaelBSPRTFsqAcmCEiByHK94EGAXkJXJZ6ayitIA/r9xBQ0s7Bbl2U6oxw82Wvc20dXZZ0gtAoos33wssASYCP4kY3gB8PcHLSlvhHb1qVyPHTx4dcDTGmMEWLumxpDf4Epr0VPV24HYR+ZCqPpDIeQ8l4X72Knc2WNIzZhiqrGlEBGaMszq9wZbo4s2PquqdwFQR+VL0eFX9STcfG3Ymjh7BiKwMu23BmGGqsqaByWPyGJGdEXQow06iizdH+v92+hJHKCTugbLWgtOYYcn63AxOoos3f+3/fyeR8x2KZo4r4Jmq2qDDMMYMsraOLt7c3cR7ji4NOpRhKVkdTpcA/wJMjVyGqn4yGctLR7PK8nlgxTb2N7dRlJcddDjGmEHy5u4mOrrUrvQCkqyb0/8MPAM8DnQmaRlpLbI7ssXTxjW/iS4AABzjSURBVAQcjTFmsGywp6UHKllJL09Vv5qkeQ8JkU9Rt6RnzPBRVdNARkiYXjKy54lNwiWrR5a/iMg5SZr3kDChMJf8nExrzGLMMLNhZwNTx+aRk2ktN4OQrKR3NS7xHRCRehFpEJH6JC0rLYlYC05jhqMq634sUElJeqpaoKohVR2hqqP8+1HJWFY6m1VaYPfqGTOMtLR3smVPkyW9ACWr9ebx3QyuA7aoqj1B3ZtZWsA9r2xld2Mrxfn2EApjhrqNuxrpUmvEEqRkNWT5FXA8sMq/nw+sBgpF5DOq+miSlptWIrsjK55hSc+Yoa5ql6vOmFVm/XcEJVl1ejuA41R1oaouBBYAm4Azgf9O0jLTTvhZWlavZ8zwsGFnI1kZwpSx1nIzKMlKehWquib8RlXXArNVdVOSlpeWSgpyKMrLonKX1esZMxxU1TRwVEk+WRnJOvSaniSreHONiNwI3OPfXwys9U9Pb0/SMtOOiFAxrsAeKGvMMLGhpoHj7MkqgUrW6cYSYCPwBf+3yQ9rB96RpGWmpYqyfDbUNKCqQYdijEmixtYOtu07wKxSq88LUlKu9FT1APC//i+aleVFmF02ijtb3mL7/gNMHG0PlzdmqNqw092qPLvM7t4KUlKu9ERkpojcLyJrRWRT+K+Hz9wqIrtEZHWM8SIi14vIRhF5PcZtEWlnznj3A1hfbUWcxgxl6/xvfM4ES3pBSlbx5u+AG4EOXHHmHcCdPXzmNuCsOOPPBmb6vyv9/NPerDJ328K6auuwxpihbP3OegpyM5lQmBt0KMNaspLeCFV9AhBV3aKq1wHvi/cBVX0a2BtnkguAO9R5ESgSkfEJizgg+TmZTB6Tx3przGLMkLauuoE5ZaMQkaBDGdaSlfRaRSQEVInIVSLyAQb+NPVyYGvE+21+WNqbM77ArvSMGcK6upQNOxuYM956YglaMjuczgM+DywEPgZ8PEnLOoKIXCkiy0RkWW1t6j+dfHbZKN7c08SBNnv0oDFD0fb9B2hs7WD2eKvPC1qyWm++4l82Ap9I0Gy3A5Mi3k/0w7pb/s3AzQCLFi1K+XsB5owfharrmeXYSUVBh2OMSbC1viRnjiW9wCU06YnIw/HGq+r5A5j9w8BVInIPcCJQp6rVA5hfyggXeayrrrekZ8wQtL66AZFDXQ+a4CT6Su9kXL3b3cBLQK9rbEXkbuAMoFhEtgHfBrIAVPUmYClwDu6m92YSdwUZuEmj8xiZnWGNWYwZotZV1zN17EjyspPVCZbprUR/A2W4TqUvBT4CPALcHdkPZyyqemkP4xX4bCKCTDWhkDCrrOBgEYgxZmhZv7OeuXZ/XkpIaEMWVe1U1b+p6seBk3BXZU+KyFWJXM5QNHv8KNZX11t3ZMYMMU2tHWzZ22w9saSIhLfeFJEcEfkg7mb0zwLXAw8lejlDzZyyAupbOthR1xJ0KMaYBHJ968LsMrtdIRUkuiHLHcA8XP3bd1S12y7FzJHCrbrW7ainvGhEwNEYYxJlnbXcTCmJvtL7KK6bsKuB50Wk3v81iIhVWMUxZ/woRGDNDttMxgwla3bUUzgii4mj7WQ2FST0Sk9V7cmI/TQyJ5NpxSNZvaMu6FCMMQm0ZnsdR0+w7sdShSWpFDJvQiFrtlvSM2aoaO/sYt3OBuaVFwYdivEs6aWQeeWj2FHXwp7G1qBDMcYkwMZdjbR1dHG03a6QMizppZB5E9zZoNXrGTM0rPYlN3allzos6aWQo33Ss3o9Y4aGNTvqGZmdwbSxI4MOxXiW9FJIYV4Wk8aMYM12u9IzZihYvb2OuRNGEQpZI5ZUYUkvxcybUGhXesYMAZ1dytrq+oMlOCY1WNJLMfPKC9myp5m6A+1Bh2KMGYA3dzfR3NZp9XkpxpJeigm38lprjVmMSWtrdoQbsVjLzVRiSS/FHH2wBacVcRqTzlZvryMnM8SMEnuGXiqxpJdiSgpyKBuVyyq7Sd2YtLZ6ez2zywrIzLDDbCqxbyMFzZ9YyGtb9wcdhjGmnzq7lNe37eeYiUVBh2KiWNJLQQsmFbF5TzP7mtqCDsUY0w8bdzXS1NbJgkmW9FKNJb0UdJz/oby2za72jElHK7fuA2DBZEt6qcaSXgqaP7EQEVhpRZzGpKWVW/czKjfTemJJQZb0UlBBbhYzx+Vb0jMmTb361n6OnVRkPbGkIEt6Keq4SaN5bet+VDXoUIwxfdDU2kFlTcPBagqTWizppagFk4vY19zOlj3NQYdijOmDVdvr6FKrz0tVlvRSVLjVlxVxGpNewr/ZY+12hZRkSS9FVZQWkJedYUnPmDSz8q39TB6Tx9j8nKBDMd2wpJeiMkLC/PJCXrWkZ0xaWbl1v92fl8Is6aWwBZOLWLejnpb2zqBDMcb0ws66FnbWt1jSS2GW9FLYwsmjaevssn44jUkTr2zeC8DxU0YHHImJxZJeCjth6hgAXn5zb8CRGGN64+U395KXncG8CfY4oVRlSS+FjR6ZzazSAl6ypGdMWnj5zb0snDLanqyQwuybSXGLp41h+ea9dHR2BR2KMSaOfU1tbKhp4MRpY4IOxcRhSS/FLZ42hqa2TtZW25PUjUll4fq8xdPGBhyJiceSXopbPM3q9YxJBy+/uZfszBDHTCwMOhQTR8okPRE5S0Q2iMhGEflaN+OXiEitiKz0f58KIs7BVjoql6lj86xez5gU9/LmvSyYVERuVkbQoZg4UiLpiUgG8EvgbGAucKmIzO1m0j+q6gL/d8ugBhmgxdPG8MrmvXR1WefTxqSixtYOVm+vs/q8NJASSQ9YDGxU1U2q2gbcA1wQcEwpY/G0sexvbqdqV2PQoRhjurF8yz669FB1hEldqZL0yoGtEe+3+WHRPiQir4vI/SIyKdbMRORKEVkmIstqa2sTHeugC589vvTmnoAjMcZ056VNe8gICcdPtpvSU12qJL3e+D9gqqoeAzwG3B5rQlW9WVUXqeqikpKSQQswWSaOHsGkMSN4pmp30KEYY7rx7MbdHDepiJE5mUGHYnqQKklvOxB55TbRDztIVfeoaqt/ewuwcJBiC5yIcNrMEp7fuJu2Drtfz5hUsqexlVXb6zi9Iv1PsIeDVEl6rwAzRWSaiGQDlwAPR04gIuMj3p4PrBvE+AJ3ekUJTW2dLN+yL+hQjDERnt24G1U4zZJeWkiJpKeqHcBVwN9xyexeVV0jIt8VkfP9ZJ8XkTUi8hrweWBJMNEG4+SjxpIZEp6uSv86SmOGkqc21DJmZDbzy+3+vHSQMgXQqroUWBo17FsRr68BrhnsuFJFQW4WC6eM5qkNtXz1rNlBh2OMAbq6lKerdnPqjGJCIQk6HNMLKXGlZ3rntIoS1lbXs6uhJehQjDHA2up6dje2Wn1eGrGkl0bCP6xnKq0VpzGpIFzd8PaK4oAjMb1lSS+NzB0/iuL8bKvXMyZFPLWhlrnjRzGuIDfoUEwvWdJLI6GQcFpFCU9uqKXdHjVkTKD2N7exfMs+Tp9lRZvpxJJemjnr6DLqDrTzwhvWO4sxQXpsbQ0dXcrZ88qCDsX0gSW9NHNaRQkjszP46+rqoEMxZlhbuqqaiaNH2K0KacaSXprJzcrgXXNK+fuaGnuaujEBqTvQzrMbd3PO/PGI2K0K6cSSXho6Z34Ze5va7Bl7xgTk8bU1tHda0WY6sqSXhs6YNY687AweWWVFnMYEYemqaiYU5rJgUlHQoZg+sqSXhnKzMnjH7HH8ffVOOu3BssYMqvqWdp6p2s3ZVrSZlizppan3zR/PnqY2nn/DblQ3ZjA9uqaGts4uzplvRZvpyJJemnrn7HEUjsjinle29jyxMSZh7nn5LaYVj7QHxqYpS3ppKjcrgw8dP5FH1+xkd2Nrzx8wxgxYZU0Dy7bs49LFk6xoM01Z0ktjHzlxEu2dygPLtwUdijHDwt0vv0V2RogPHT8x6FBMP1nSS2MzxhWweOoY7n75LbqsQYsxSdXS3skDy7fx3nlljM3PCToc00+W9NLcR06czOY9zby4ybolMyaZlq6qpr6lg0sXTwo6FDMAlvTS3FnzyijKy+L2FzYHHYoxQ5aqcvsLW5hWPJKTp48NOhwzAJb00lxuVgaXnzyVv6+pYV11fdDhGDMkPV21m9e27udTb59mDVjSnCW9IeCKU6ZRkJPJDf+oCjoUY4YcVeXnj1cyoTCXDy+0os10Z0lvCCjMy2LJKVNZumon63fa1Z4xifRM1W5WvLWfz7xjBtmZdshMd/YNDhFXnDqN/JxMbnhiY9ChGDNkqCo/f6KK8YW5XLTIblMYCizpDRFFedl84pSpPLKqmuVb7OkLxiTC31bvZPmWffzbGUeRk5kRdDgmASzpDSGfPv0oyotG8LUHVtHWYc/aM2Yg6g608+2H1zB3/CguXTw56HBMgljSG0JG5mTy/ffPo2pXIzc99UbQ4RiT1v7rb+vZ3djKf33oGDIz7FA5VNg3OcS8Y/Y4zjt2Ar/4x0Y27moIOhxj0tJLm/bwh5fe4pOnTGP+xMKgwzEJZElvCPrWuXPJz83kyjuWU9fcHnQ4xqSVHfsP8Nk/vMrkMXl86T0VQYdjEsyS3hBUUpDDTR9dyNZ9zfzbH5bT3mn1e8b0RlNrB1fcvozW9k5u+fgi8rIzgw7JJJglvSFq8bQx/PCDx/Dcxj1c+9Aqe8K6MT1oae/k83e/yoad9dzwkeOoKC0IOiSTBHYaM4RduHAib+1t5vonqtjX3M71lxzHiGxrdm1MtP3NbfzLHct4ZfM+vvf+eZwxa1zQIZkksSu9Ie5LZ1Zw3XlzeXxdDZf85kW27GkKOiRjUsq66no+eOPzvLa1jusvPY6PnTQl6JBMElnSGwaWnDKNX390IRtrGnjPT5/m+ieqaGnvDDosYwLV2NrB9/+ylnNveJb9ze3c+akTOf/YCUGHZZJMVFOjrkdEzgJ+DmQAt6jqj6LG5wB3AAuBPcDFqrq5p/kuWrRIly1blviA09DOuha+98haHnm9muL8bC4+YRKXnDCZSWPygg5tWFBVWju6ONDmTjgyMoQMETJC7i8zJNaD/yDYuKuBu156iweWb6OhtYNLTpjMV8+aRVFedtChpQwRWa6qi4KOIxlSIumJSAZQCZwJbANeAS5V1bUR0/wbcIyqflpELgE+oKoX9zRvS3pHev6N3dz67Gb+sb6GLoXpxSN524yxzB1fyPSSkZQXjWDUiCzyczLJCA3eQVhVUYUuVbr8/0Pv3TCNGhf5PiRCKASZoRAZ/nU4oYSTS7ykoqq0dyrtnV20dXTR3tlFS3sXze0dNLV2cqCtk6a2jsP/t3bS3N5Bc2snzW2dNLd1RP33r1s7aW7v7LFBUUggJzODkTmZ5Oe4/+515qFh2YeG5eVkuHHZh17nZR8aNyIrw2+L3n2Pqkpnl9Lp/3d0KZ2dh95H/nV0ue8l/D68/TPEbeeMkBAS/HD3HYTk0HcSivheQuKSfm/j7I3mtg72NLbx1t5mNtU2smp7Hc+/sYdt+w6QlSGcNW88nzp1GsdOKkrYMocKS3rJDkLkZOA6VX2vf38NgKr+MGKav/tpXhCRTGAnUKI9rIAlvdi27z/AX1dV89zG3bz85l6a2o4s8szPyaQgNzNu7/LaiwTV1dV9QlMOf59sIriDa8QVliq0dXTR1s9bO7IzQ4zMziAvO5O87Az/51/nZJKXlUFezqHhI7IyCAkuoYQTTKcefN/S7pJqY2snTa0dNLZ20OT/wsMO9LF4Orze7ooy5JJRSA5LYOHXQQufpIQTaMhfBWdEfG/h/+7qGDq73AlLW2f4ZKWTlvbDv8/CEVmcNH0Mp8wo5ux54ykpyAloDVPfUE56qdJ6sxzYGvF+G3BirGlUtUNE6oCxwO7omYnIlcCVAJMnW595sZQXjeBTb5/Op94+na4uZUfdAd6obaKmroX6lnYaWjpoaOmgvqWdjh4SQsgfiMJn9hLxOiQg4g5OPU0TOjjdofeHpo/4vL9qEw6NCyfXw69C3PAOf7XS1XXkVUtIhKxMIScjRFZGiOxM95eVESInM8TInMzDE1m2uwIbkZ1BXlZGIF1UdXYpzW3uCrTJX0k2tnbQ3Nbh/7vk2NLeSWcXdHZ1HUxs4f9dqgcTR0YodPBK61ByPLzoNTwuJEJmRnTikSNOeroirgTD46KvDsPfUWdXl/sf8R11RcUa/n/4d+mWk5khZIW/vwwhJyuD0XnZjBmZxcTReRxVkk/pqBwrPjYpk/QSSlVvBm4Gd6UXcDhpIRQSJo7OY+Joq99LBxkhoSA3i4LcrKBDMSatpErrze1A5COJJ/ph3U7jizcLcQ1ajDHGmF5JlaT3CjBTRKaJSDZwCfBw1DQPAx/3ry8E/tFTfZ4xxhgTKSWKN30d3VXA33G3LNyqqmtE5LvAMlV9GPgt8HsR2QjsxSVGY4wxptdSIukBqOpSYGnUsG9FvG4BPjzYcRljjBk6UqV40xhjjEk6S3rGGGOGDUt6xhhjhg1LesYYY4aNlOiGLJlEpBbY0s+PF9NNjy9pxOIPVjrHn86xg8U/UFNUtSTA5SfNkE96AyEiy9K5/zmLP1jpHH86xw4Wv4nNijeNMcYMG5b0jDHGDBuW9OK7OegABsjiD1Y6x5/OsYPFb2KwOj1jjDHDhl3pGWOMGTYs6RljjBk2LOl1Q0TOEpENIrJRRL4WdDw9EZFJIvJPEVkrImtE5Go/fIyIPCYiVf7/6KBjjUdEMkTkVRH5i38/TURe8t/DH/1jp1KSiBSJyP0isl5E1onIyem0/UXki37fWS0id4tIbipvfxG5VUR2icjqiGHdbm9xrvfr8bqIHB9c5Adj7S7+//H7z+si8pCIFEWMu8bHv0FE3htM1EODJb0oIpIB/BI4G5gLXCoic4ONqkcdwJdVdS5wEvBZH/PXgCdUdSbwhH+fyq4G1kW8/y/gp6o6A9gHXBFIVL3zc+BvqjobOBa3Hmmx/UWkHPg8sEhV5+Ee73UJqb39bwPOihoWa3ufDcz0f1cCNw5SjPHcxpHxPwbMU9VjgErgGgD/W74EONp/5lf+OGX6wZLekRYDG1V1k6q2AfcAFwQcU1yqWq2qK/zrBtwBtxwX9+1+stuB9wcTYc9EZCLwPuAW/16AdwL3+0lSNn4RKQROwz3zEVVtU9X9pNH2xz1mbISIZAJ5QDUpvP1V9WncczUjxdreFwB3qPMiUCQi4wcn0u51F7+qPqqqHf7ti8BE//oC4B5VbVXVN4GNuOOU6QdLekcqB7ZGvN/mh6UFEZkKHAe8BJSqarUftRMoDSis3vgZ8B9Al38/FtgfcRBI5e9hGlAL/M4Xz94iIiNJk+2vqtuBHwNv4ZJdHbCc9Nn+YbG2dzr+pj8J/NW/Tsf4U5YlvSFERPKBB4AvqGp95Dh196ak5P0pInIusEtVlwcdSz9lAscDN6rqcUATUUWZKb79R+OuJqYBE4CRHFn0llZSeXv3RESuxVVZ3BV0LEORJb0jbQcmRbyf6IelNBHJwiW8u1T1QT+4JlyM4//vCiq+HpwCnC8im3HFye/E1ZEV+eI2SO3vYRuwTVVf8u/vxyXBdNn+7wbeVNVaVW0HHsR9J+my/cNibe+0+U2LyBLgXOAyPXQTddrEnw4s6R3pFWCmb7mWjatAfjjgmOLy9V+/Bdap6k8iRj0MfNy//jjw58GOrTdU9RpVnaiqU3Hb+x+qehnwT+BCP1kqx78T2Cois/ygdwFrSZPtjyvWPElE8vy+FI4/LbZ/hFjb+2Hgct+K8ySgLqIYNGWIyFm4Iv7zVbU5YtTDwCUikiMi03ANcl4OIsYhQVXtL+oPOAfXeuoN4Nqg4+lFvKfiinJeB1b6v3Nw9WJPAFXA48CYoGPtxbqcAfzFv56O+3FvBO4DcoKOL07cC4Bl/jv4EzA6nbY/8B1gPbAa+D2Qk8rbH7gbV//YjrvSviLW9gYE1yL7DWAVrpVqKsa/EVd3F/4N3xQx/bU+/g3A2UHHn85/1g2ZMcaYYcOKN40xxgwblvSMMcYMG5b0jDHGDBuW9IwxxgwblvSMMcYMG5b0zKATkU4RWRnxNzXomBJFRI4Tkd/612eISF3Een4rYrojetn3w2P2tB8xzQQRuT96eJyYNovIAxHvLxSR2/q1gvGXc0u8ztlFZImITIh4f4+IzEx0HMbEY0nPBOGAqi6I+NscHuFvIE7n/fLrwPUR75+JWM/vRgy/je67+uq2p/1IqrpDVS884pPxLUz200JU9VOqujbOJEtw3ZyF3Yi7GduYQZPOBxczRIjIVP+csDtwN0dPEpF/F5FX/BXPdyKmvVZEKkXkWXHPffuKH/6kiCzyr4t9l2bhZ/T9T8S8/tUPP8N/JvwMvLt8bySIyAki8ryIvCYiL4tIgYg8LSILIuJ4VkSOjVqPAuAYVX2tp3XW7p8SgMbuaT96e632r4/2Ma706xfryul/cTc4R84nJO7ZcyUR7zeKSInvkegFEVklIt8XkcaI7faXiHn8wneddfA78Nv8NnHP5lsl7ll9FwKLgLt8rCOAZ4B3y6GuzoxJOkt6JggjIor8HvLDZgK/UtWjgVn+/WJcTycLReQ0EVmI66ZsAa7HmRN6sawrcN1OneCn/xfflRO4p1F8AffcxOnAKeK6nvsjcLWqHovrl/IArpu3JQAiUgHkdpPcFuGSdqSTffL8q4gc3Yt4I0X2tB/Lp4Gfq+oCv/xtMaa7FzheRGaEB6hqF3AncJkf9G7gNVWtxfV9eqOqzsf1HNIXC4ByVZ3nP/87Vb0f12PNZf6q94Bf/kbc8weNGRSW9EwQIos3P+CHbVH3rDOA9/i/V4EVwGxcEnw78JCqNqt7ikRv+kR9D67fxZW4xy2N9fMCeFlVt/mD70pgKi7hVqvqKwCqWu+vvO4DzhXXsfcnccWT0cbjHjEUtgKY4pPnDbjuyXpFet/T/gvA10Xkq35ZB2JM1wn8D0cWl94KXO5ffxL4nX99Cq6rLHDdkvXFJmC6iNwgrj/J+jjT7uLwIk9jksqSnkkVTRGvBfhhRGKcoaq/7eHzHRzan3Oj5vW5iHlNU9VH/bjWiOk6cY8I6pa6DoAfwz2C5yK6T0YHIpftE2ajf70UyBKR4h7WI1ZP+7Hi+gNwvl/2UhF5Z5zJf4972O3BHvtVdSvu6QTvxF1ZR15ZdrfsyO0Mh2/r8Dz34a7ensRdid4SJ6ZcH7sxg8KSnklFfwc+Ke75gIhIuYiMA54G3i8iI3z92XkRn9kMLPSvL4ya12f8FRoiUiHuAa+xbADGi8gJfvqCiDqnW3CNVF7xB/Zo64CDxYciUhZRT7gY93vbE2/FJXZP+7Gmnw5sUtXrcU8VOCbWtOoeG/RT4ItRo27BFXPep6qdfthzuKJkOFT8CbAFmCuux/8i3BMZomMqBkKq+gDwDdxjlgAagIKoySs4skjYmKSxpGdSjr8S+wPwgoiswj2frkBVV+Dq217DXZG8EvGxH+OS26tA5NXULbjH5KzwjT9+TfwrujbgYuAGEXkNd3WX68ctxxXV/S7GZ9cDhT4hg0u+q/18rgcuCV+5icjduKLJWSKyTUSu8J/5BS4xPObrPG+Kv7W4yC9jJTAPuKOH6X/Lkev/MJAftV5XA5/12//gU7r9leG9uER1L64IOlo58KSP6U4OFaneBtwUbsgiIqW4ou6dPcRsTMLYUxZM2hKR64BGVf3xIC1vAq7IbravB+xumi8CDaoar0gvpfhWrz9V1bfHmaZRVfMTvNwvAvW9KLo2JmHsSs+YXhCRy3ENYa6NlfC8Gzm8rjClicjXgAfo5n7AQbAfuD2A5ZphzK70jDHGDBt2pWeMMWbYsKRnjDFm2LCkZ4wxZtiwpGeMMWbYsKRnjDFm2Ph/2hPcK/6KS8EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "f, H_im = sig.freqz(h)\n", + "posfreq = np.square(H[0:512//N])\n", + "negfreq = np.flipud(np.square(H[0:512//N]))\n", + "plt.plot((np.abs(posfreq) + np.abs(negfreq)))\n", + "plt.xlabel('Frequency (512 is Nyquist)')\n", + "plt.ylabel('Magnitude')\n", + "plt.title('Unity Condition, Sum of Squared Magnitude of 2 Neighboring Subbands')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(63,)" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "b.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEWCAYAAACqitpwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nO3deXycZbn4/8+VTPakSZomXdOk0JRSCrSQtqAIiKBFloIilIMCfjlf9Kscj4eDCi6oHDgH1N9BPeKCiIKsPSxSZKkgsimUpjultE3bpEmXrE2bpVkmc/3+eJ4p0yF7ZuaZSa/36zWvzNzPdt/JZK65l+e+RVUxxhhjIinJ6wwYY4wZeyy4GGOMiTgLLsYYYyLOgosxxpiIs+BijDEm4iy4GGOMiTgLLiZuiMirIvLPA2z/tYh8L5Z5ShQiMl1E2kQkeYTHV4nIuZHOVzwRkU0icrbX+ThaWHAxfXI/bA65H1j7ROQPIpIdw+tfKyJvhqap6pdV9T+icK1SEVG3rG1u2W8O2S4i8g0R2eb+TnaJyH+KSGrIPn9wz7Ek7Nx3u+nXhpSrN+RabSLyiz7ydKWIbA5Le6mftJtVdZeqZqtqb4R+LRElIj8QkZ6wcn8zitf7g4jcHpqmqieo6qvRuqY5kgUXM5CLVDUbmAfMB27xOD/RlueW90rgVhFZ7Kb/HLgeuBrIAc4HzgUeCzt+q7sPACLiAy4Htoft95YbCIKPG/rIy+vAbBEpDDnXyUBGWNrp7r6J4PGwcv/I6wyZ6LHgYgalqvuAFThBBgAROU1E/iEiLSKyPrS5wf12vkNEWkVkp4hc5ab/QEQeCtkvWGPwhV5PRI4Hfg2c7n7DbXHTD38bFZGzRaRWRP5dROpFZK+IfDHkHAUi8qyIHBSRVSJye3hNaIDyvgVsAuaKSBnwFeAqVX1LVf2qugn4LHCBiJwVcuizwBkiku++XgxsAPYN5bphedgN7ADOdJNOcfP0WlhaErAq/HfpNjH+h4j83f07/EVEJoT8fr4gItUi0iQi3wm9toikichPRWSP+/ipiKS5214Tkc+6zz/qXvMC9/UnRGTdcMo52HtiCOU4I+R9WOO+964HrgK+6b5/nnX3Pdz0N0gZB3xvmaGx4GIGJSLTcL6tV7qvpwLPAbcD44GbgCdFpFBEsnC+6Z+vqjnAR4BhfeCo6mbgy3zwDT+vn10nAbnAVOA64J6QD/Z7gHZ3n2vcx1DKKiLyUeAEYC3wCaBWVd8Jy2MN8DbwyZDkTuAZYKn7+mrgwaFctx+v80EgORN4A3gzLO1tVe3p5/h/Ar4IFAGpOH8nRGQO8CvgC8AUoACYFnLcd4DTcL5MnAwsBL7rbnsNONt9fhZHBsCz3O2R1l85SoAXgP8BCt38rlPVe4GHgR+575+L+jjnQGWEgd9bZggsuJiB/ElEWoEaoB74vpv+eeB5VX1eVQOq+hJQAXza3R7A+dafoap73W/60dAD3KaqPar6PNAGHCdOp/Znge+raoeqvgc8MITzNQLNwH3Azar6V2ACsLef/ffifKiFehC4WkTycD5s/9THcae537SDj9P6OX9oLeVjOMHljbC0gT7Mf6+qW1X1ELCMD2qelwF/VtXXVbUL+B7O3yzoKpzfa72qNgA/xAlEwTwFa2tnAv8V8nqw4HJ5WLmnDLDvUMrxT8DLqvqo+x5oUtWhfpEZqIzQz3triOc2WHAxA7vErX2cDczG+aAFKAE+F/pBAZwBTFbVduAKnJrHXhF5TkRmRyl/TarqD3ndAWTjfOD7cIJiUOjz/kxQ1XxVPV5Vf+6mNQKT+9l/srv9MFV9073+d3A+wA/1cdzbqpoX8ni7n/O/DpzkfmM+Dacm9z4w2U07g4H7W0Kb44K/G3BqK4d/H+7frClk3ylAdcjrajcN4C1glohMxPmQfxAodpuqFg6Sn2Vh5d4zwL5DKUcxH+7PGqqBygj9v7fMEFlwMYNS1deAPwA/cZNqgD+GfVBkqeqd7v4rVPU8nA/f94Hfuse1A5khp5400GVHkeUGwM+RTT3FIzzXKzgfngtDE0WkGOcD/9U+jnkI+HdG1ySGqu4A9uAMJtilqm3uprfctGycprnh2kvI70NEMnGaxoL24HyBCJrupqGqHcBq4F+Bd1W1G/gHcCOwXVWPCLZDMJz3RLga4Nh+tg32/um3jCYyLLiYofopcJ6InIzz4XmRiHxKRJJFJN3tBJ0mIhNFZInb99KF05wQbHJZB5wpzj0ZuQw8+qwOmCYhw32Hyh2O+xTwAxHJdGtOVw9yWH/n2oozuOBhcQYxJIvICcCTOB+qL/dx2M+B84jMKK43cD643whJe9NNq+inZjSYJ4AL3c7wVOA2jvwseBT4rtuHNgG4FedvHvQacAMfNIG9GvZ6OIbzngj3MHCuiFwuIj5xBnEEm8zqgGMGOHawMppRsuBihsRtl34QuNXtzF4CfBunllADfAPn/ZSE88G3B6f/4izg/7nneAl4HGcE1WrgzwNc8hWc0VH7RGS434bB+bDLxWlS+SPOh0nXCM4TPNd9OB8+HcC7OM0ol6hqIHxnVW1W1b9qZBZLeg2nIzt0pNsbbtqIgpfbB/ZV4BGcWsx+oDZkl9tx+tA2ABuBNW5aaJ5yQq4f/no4eRnOeyL82F04/Xz/jvNeW4fTOQ/wO2CO22zbV7/XYGU0oyS2WJg5GojIXcAkVR3SqLFBzvVD4FLgTFVtGXXmjBmDrOZixiQRmS0iJ7lDixfiDCd9OhLnVtXvA/fi9LkYY/pgNRczJonIApymsCk47e/3AndGqKnKGDMICy7GGGMizprFjDHGRJxv8F3GvgkTJmhpaanX2TDGmISyevXqRlUNn6UCsOACQGlpKRUVFV5nwxhjEoqIVPe3zZrFjDHGRJwFF2OMMRFnwcUYY0zEWXAxxhgTcRZcjDHGRJynwUVEFovIFhGpFJGb+9h+poisERG/iFwWtq1XRNa5j+Uh6TNEZKV7zsdHMquuMcaY0fEsuLirBd6Ds3zuHOBKd/nVULuAa3Fmbw13SFXnuY+LQ9LvAu5W1Zk4s71eF/HMG2OMGZCXNZeFQKWq7nAXHHoMZxr3w1S1SlU3cOQSrP0SEQHOwVmvApylbS+JXJaNiZ2Obj/LKmro7On1OivGDJuXwWUqRy49W+umDVW6iFSIyNsiEgwgBUBLyPKk/Z5TRK53j69oaGgYbt6Nibpbn9nEN5/YwJ0vvO91VowZtkTu0C9R1XLgn4Cfikh/y532SVXvVdVyVS0vLOxz9gJjPPPiu3t5YnUt08dn8od/VPHGNvsCZBKLl8FlN0euaz7NTRsSVd3t/tyBs8zqfKAJyBOR4LQ2wzqnMfGgvrWTW57ayIlTc3nua2cwsyibm/53PS0d3V5nzZgh8zK4rALK3NFdqcBSYPkgxwAgIvkikuY+nwB8FHjPXavjb0BwZNk1wDMRz7kxUaKqfPOJDXR093L3FSeTk57CT6+YR1NbN9/907vYEhkmUXgWXNx+kRuAFcBmYJmqbhKR20TkYnAWfBKRWuBzwG9EZJN7+PFAhYisxwkmd6rqe+62bwE3ikglTh/M72JXKmNG5+GVu3h1SwPf/vTxzCzKAWDu1Fz+7bxZ/HnDXpav3+NxDo0ZGlssDCgvL1ebFdl4bUdDGxf8/E3KS/N54IsLSUqSw9v8vQGuuPdttta18uLXz2RqXoaHOTXGISKr3b7vD0nkDn1jxgx/b4B/W7aeVF8SP77s5CMCC4AvOYn/vvxkAgHlpmXrCQTsS6GJbxZcjIkDK3c2s76mhe9dOIdJuel97lNSkMU3F8/mrR1NbNx9IMY5NGZ4LLgYEwdW7mgiSeBTJ0wccL/z505y9t/ZFItsGTNiFlyMiQNv72zmhCm55KSnDLhf0bh0ZkzIYuWO5hjlzJiRseBijMc6e3pZV9PCohnjh7T/ohnjeaeqmV7rdzFxzIKLMR5bV9NCtz/AomMKhrT/omPG09rpZ/Peg1HOmTEjZ8HFGI+t3NGMCCwsHWrNxQlCK3da05iJXxZcjPHYyp1NzJ40jtzMgftbgqbkZVA8PoOVO6xT38QvCy7GeKjbH2DNrv1D7m8JWjSjgHeqmu1+FxO3LLgY46ENtS109gQ47ZjhBpfxtHT0sLW+NUo5M2Z0LLgY46Fgv8nCGUPrzA86ze38f8f6XUycsuBijIdW7mxm1sRsxmelDuu4afkZTMlNt/tdTNyy4GKMR/y9AVZXNR8e/TUcIsKiYwpYubPJpuE3ccmCizEeeXfPQdq7e1k0zP6WoEUzxtPY1s32hvYI58yY0bPgYoxHgkOJFw5zpFhQ8KZLm2fMxCMLLsZ4ZOXOZo4pzKIop+9ZkAdTWpBJUU6a9buYuGTBxRgP9AaUVTubh31/SygRYeGM8dbvYuKSBRdjPLB570Fau/wj6swPteiYAuoOdlHd1BGhnBkTGRZcjPFA8P6WkXbmB53m1nzsfhcTbyy4GOOBlTuamD4+k8m5GaM6z8yibAqyUnnbOvVNnPE0uIjIYhHZIiKVInJzH9vPFJE1IuIXkctC0ueJyFsisklENojIFSHb/iAiO0VknfuYF6vyGDMUgYDyTtXo+luCDve7WKe+iTOeBRcRSQbuAc4H5gBXisicsN12AdcCj4SldwBXq+oJwGLgpyKSF7L9G6o6z32si0oBjBmh6uYOWjp6KC/Nj8j5ykvHs7vlEA2tXRE5nzGR4GXNZSFQqao7VLUbeAxYErqDqlap6gYgEJa+VVW3uc/3APVAYWyybczobKtzJps8btK4iJzvuIk5znltEksTR7wMLlOBmpDXtW7asIjIQiAV2B6SfIfbXHa3iKT1c9z1IlIhIhUNDQ3DvawxI7atvg1w+ksiYdZE5zzb6toicj5jIiGhO/RFZDLwR+CLqhqs3dwCzAYWAOOBb/V1rKreq6rlqlpeWGiVHhM72+pamZqXQXaaLyLnK8xJY1y6z2ouJq54GVx2A8Uhr6e5aUMiIuOA54DvqOrbwXRV3auOLuD3OM1vxsSNbfVtEau1gNOpXzYxx2ouJq54GVxWAWUiMkNEUoGlwPKhHOju/zTwoKo+EbZtsvtTgEuAdyOaa2NGoTegVNa3HW7KipRZE7MPN7cZEw88Cy6q6gduAFYAm4FlqrpJRG4TkYsBRGSBiNQCnwN+IyKb3MMvB84Eru1jyPHDIrIR2AhMAG6PYbGMGVDt/g66/AHKinIiet6ZRTk0t3fT1GYjxkx8iEyj7wip6vPA82Fpt4Y8X4XTXBZ+3EPAQ/2c85wIZ9OYiAk2Xc2McM2lzG1m21rXxunZfY5hMSamErpD35hEE1zzviyCfS4As9zhyJXWqW/ihAUXY2Kosq6Nybnp5KSnRPS8E8elkZPms34XEzcsuBgTQ5EeKRYkIsycmM3WOqu5mPhgwcWYGAkcHikW2c78oFlFOVRazcXECQsuxsTI7pZDHOrpjXh/S1DZxGwa27ppbu+OyvmNGQ4LLsbESPAO+rIIjxQLCja3bbOmMRMHLLgYEyNbg8OQI3yPS9CswxNYWtOY8Z4FF2NiZFtdGxPHpZGbEdmRYkGTc9PJTvNZzcXEBQsuxsTItvrWqHXmgztirMimgTHxwYKLMTEQHCkWjWHIocosuJg4YcHFmBjYc+AQHd29EZ9TLFzZxGwaWrto6bARY8ZbFlyMiYHgnGKRng05XJl16ps4YcHFmBgIDkOORbMYYHfqG89ZcDEmBrbVtVGYk0ZeZmpUrzMlN4PM1GRbOMx4zoKLMTGwNQoLhPUlKUkoK8q2aWCM5yy4GBNlqkplXWvUO/ODZhblWLOY8ZwFF2OibO+BTtq7e6Pe3xJUNjGb+tYuDnT0xOR6xvTFgosxURasRUTzBspQwea3bbZwmPGQBRdjoizY/xGt2ZDDBZvfbDiy8ZIFF2OibFtdGxOyU8nPiu5IsaCpeRlkpNiIMeMtT4OLiCwWkS0iUikiN/ex/UwRWSMifhG5LGzbNSKyzX1cE5J+qohsdM/5cxGRWJTFmP5srY9dZz44I8acOcasWcx4x7PgIiLJwD3A+cAc4EoRmRO22y7gWuCRsGPHA98HFgELge+LSL67+VfA/wXK3MfiKBXBmEE5I8XaoraGS3/KimzJY+MtL2suC4FKVd2hqt3AY8CS0B1UtUpVNwCBsGM/Bbykqs2quh94CVgsIpOBcar6tqoq8CBwSdRLYkw/Gtu6ae3yc8yErJhe99iibOoOdtHR7Y/pdY0J8jK4TAVqQl7XummjOXaq+3zQc4rI9SJSISIVDQ0NQ860McNR3dQOQGmMg0tJQaZ7/Y6YXteYoKO2Q19V71XVclUtLyws9Do7Zoza2egGl4LYBpfg9arc6xsTa14Gl91AccjraW7aaI7d7T4fyTmNibjqpg6Sk4Sp+Rkxve50t+ZSZTUX4xEvg8sqoExEZohIKrAUWD7EY1cAnxSRfLcj/5PAClXdCxwUkdPcUWJXA89EI/PGDEVVUzvT8jNISY7tv9q49BQKslIPN8sZE2ueBRdV9QM34ASKzcAyVd0kIreJyMUAIrJARGqBzwG/EZFN7rHNwH/gBKhVwG1uGsBXgPuASmA78EIMi2XMEaqbOmLeJBZUOiGLKgsuxiM+Ly+uqs8Dz4el3RryfBVHNnOF7nc/cH8f6RXA3Mjm1JjhU1Wqmto5ZXqeJ9cvKcjkre1NnlzbmKO2Q9+YaNvf0UNrp58Sr2ouBVnsPdBJZ0+vJ9c3RzcLLsZEyeGRYhMyPbm+DUc2XrLgYkyUBDvTvay5ANbvYjxhwcWYKKlq6iBJoDjfm5pLMLjYiDHjBQsuxkRJdVM7U/MzSPV582+Wm5lCfmaK3etiPGHBxZgoqfJwGHJQSUGW1VyMJyy4GBMl1U3thzvVvVJakElVo9VcTOxZcDEmClo6umnp6ImLmsueA4dsOLKJOQsuxkRBsJ/Dq5FiQaUTMlGF2v1WezGxZcHFmCg4PNW+x81iJYdnR7bgYmLLgosxUVDV2IEIFI/3NrjMsHtdjEcsuBgTBdVN7UzJzSA9JdnTfORlpjAu3Wd36ZuYs+BiTBTsjIORYgAiYrMjG09YcDEmCqqbOjzvzA8qKbDgYmLPgosxEXbgUA/N7d2ed+YHlRZksnv/Ibr9Aa+zYo4iFlyMibBdcTIMOaikIIuADUc2MWbBxZgICzZBzZgQH8FlxgSbet/EngUXYyIseI/LdI+HIQeV2HBk4wELLsZE2M7GDiaNSycj1dthyEEFWalkp/moarTgYmLH0+AiIotFZIuIVIrIzX1sTxORx93tK0Wk1E2/SkTWhTwCIjLP3faqe87gtqLYlsoc7eJhwspQIkJJQaZNvW9iyrPgIiLJwD3A+cAc4EoRmRO223XAflWdCdwN3AWgqg+r6jxVnQd8AdipqutCjrsquF1V66NeGGNCxMNU++FKbep9E2Ne1lwWApWqukNVu4HHgCVh+ywBHnCfPwF8QkQkbJ8r3WON8Vxbl5/Gti5KJsRPzQWgpCCT2v2H6Om14cgmNrwMLlOBmpDXtW5an/uoqh84ABSE7XMF8GhY2u/dJrHv9RGMjImaYO1gRrzVXCZk4Q8oe1oOeZ0Vc5RI6A59EVkEdKjquyHJV6nqicDH3McX+jn2ehGpEJGKhoaGGOTWHA2q4+wel6DSwyPGrN/FxMaQg4uIRLqevxsoDnk9zU3rcx8R8QG5QFPI9qWE1VpUdbf7sxV4BKf57UNU9V5VLVfV8sLCwlEUw5gP7HRHZMVThz58MPW/jRgzsTJocBGRj4jIe8D77uuTReSXEbj2KqBMRGaISCpOoFgets9y4Br3+WXAK6qqbj6SgMsJ6W8REZ+ITHCfpwAXAu9iTIxUN7VTmJNGVprP66wcoTAnjYyUZLvXxcTMUP4D7gY+hfvBr6rrReTM0V5YVf0icgOwAkgG7lfVTSJyG1ChqsuB3wF/FJFKoBknAAWdCdSo6o6QtDRghRtYkoGXgd+ONq/GDJUzUiy+ai3wwXBku0vfxMqQvl6pak1Yv3hEFuRW1eeB58PSbg153gl8rp9jXwVOC0trB06NRN6MGYnqpnY+VhafzaylBVlsrWv1OhvmKDGUPpcaEfkIoCKSIiI3AZujnC9jEk5Ht5+6g11xM6dYuNIJWdTs76A3oF5nxRwFhhJcvgx8FWdY8G5gnvvaGBPig5Fi8dcsBk6nfk+vDUc2sTFos5iqNgJXxSAvxiS04D0u8XZ3flDoBJbFcTKpphm7Bg0uIvJ74EP1aFX9P1HJkTEJKngPyfR4rbm4swZUNXXwsTKPM2PGvKF06P855Hk6cCmwJzrZMSZxVTe1U5CVyrj0FK+z0qeJOemk+ZKotntdTAwMpVnsydDXIvIo8GbUcmRMgqpq7Ijb/haApCSbHdnEzkimfykDbBp7Y8JUN7VTGqcjxYJsdmQTK0Ppc2nF6XMR9+c+4FtRzpcxCaWzp5c9BzrjtjM/qHRCFq9ubSAQUJKSbE5XEz1DaRbLiUVGjElku5rjexhyUElBJt3+AHsPdjI1L8Pr7JgxrN/gIiKnDHSgqq6JfHaMSUzBCSHjvubi5q+6sd2Ci4mqgWou/98A2xQ4J8J5MSZhBW+gjPfgEqxZVTV18JGZHmfGjGn9BhdV/XgsM2JMIqtqaicvM4XczPgchhw0OTeD1OQk69Q3UTekiStFZC7OOvfpwTRVfTBamTIm0VQ3dcR9rQUgOUmYXpBpU++bqBvKaLHvA2fjBJfngfNx7nOx4GKMa2djOwtK873OxpCU2tT7JgaGcp/LZcAngH2q+kXgZJwVIY0xQJe/lz0HDsXd0sb9KSnIoqqpHXfdPWOiYijBpVNVA4BfRMYB9Ry5PLExR7Wa5kOofjB3V7wrLciksydA3cEur7NixrB+g4uI3CMiZwDviEgezoqOq4E1wFsxyp8xcS/YOZ5INRfA+l1MVA3U57IV+DEwBWgHHgXOA8ap6oYY5M2YhFCVIMOQgw7f69LUzmnHFHicGzNW9VtzUdWfqerpOGvVNwH3Ay8Cl4qITdhtjKuqsZ1x6T7y43wYctCUvHRSksUmsDRRNWifi6pWq+pdqjofuBK4BHg/6jkzJkFUuRNWiiTGXF2+5CSK8zPtXhcTVYMGFxHxichFIvIw8AKwBfhMJC4uIotFZIuIVIrIzX1sTxORx93tK0Wk1E0vFZFDIrLOffw65JhTRWSje8zPJVH+403Cqm7qSJj+lqCSgkyqGq3mYqJnoA7980TkfqAW+L/Ac8CxqrpUVZ8Z7YVFJBm4B+e+mTnAlSIyJ2y364D9qjoTuBu4K2TbdlWd5z6+HJL+Kze/Ze5j8Wjzakx/uv0Bavd3UBrnE1aGK3Gn3rfhyCZaBqq53AL8AzheVS9W1UdUNZL16IVAparuUNVu4DFgSdg+S4AH3OdPAJ8YqCYiIpNxBhy8rc5/zYM4zXjGRMXulkMENHFGigWVFmTS3t1LQ5sNRzbRMVCH/jmqep+q7o/StacCNSGva920PvdRVT9wAAgOb5khImtF5DUR+VjI/rWDnBMAEbleRCpEpKKhoWF0JTFHreBw3oSruUwIjhizpjETHSNZiTIe7AWmu4MMbgQecW/wHDJVvVdVy1W1vLCwMCqZNGPf4an243wFynAzgve6NFqnvokOL4PLbo6803+am9bnPiLiw5l2pklVu1S1CUBVVwPbgVnu/tMGOacxEVPd1EF2mo+CrFSvszIsU/MzSE4Sq7mYqPEyuKwCykRkhoikAkuB5WH7LAeucZ9fBryiqioihe6AAETkGJyO+x2quhc4KCKnuX0zVwOjHnxgTH+qmtopKchMmGHIQSnJSUzLz7C79E3UDGnK/WhQVb+I3ACsAJKB+1V1k4jcBlSo6nLgd8AfRaQSaMYJQODc2HmbiPQAAeDLqtrsbvsK8AcgA2fo9AuxKpM5+lQ3dTBn8rBaZOOGM2LMai4mOjwLLgCq+jzONP6habeGPO8EPtfHcU8CT/ZzzgpgbmRzasyH+XsD1DR3cP7cSV5nZURKCzJZu2s/qppwNS8T/xK1Q98Yz+1p6cQf0ISZUyxcSUEWrZ1+mtu7vc6KGYMsuBgzQjsPz4acWMOQg4LDp22OMRMNFlyMGaHg3FwzEmwYclDphA9mRzYm0iy4GDNCVY0dZKQkU5iT5nVWRmRafgZJYjUXEx0WXIwZoeoEHYYclOZLZkpehtVcTFRYcDFmhHY2tidsZ37QjAlZ7LS79E0UWHAxZgS6/L1UN3cwsyjb66yMyrGF2VTWt9nsyCbiLLgYMwJVjR30BpSyiYkdXGYWZdPR3cueA51eZ8WMMRZcjBmByvo2wPnmn8iCNa9geYyJFAsuxozAtvpWRBI/uJS5wWVbXavHOTFjjQUXY0agsr6NafkZZKQme52VUSnITiM/M4XtDVZzMZFlwcWYEaisb6OsKMfrbEREWVEO2+osuJjIsuBizDD5ewPsaGxP+JFiQccWZbPNRoyZCLPgYsww1ew/RLc/MGaCS1lRNgcO9dDYZhNYmsix4GLMMAVHVpWNleAy0UaMmciz4GLMMG2rd0ZWHTtGgssHw5FtxJiJHAsuxgxTZX0bk8alMy49xeusRMSkcelkp/ms5mIiyoKLMcNUWd82ZvpbAETkcKe+MZFiwcWYYVDVMRdcwOk/spqLiSQLLsYMw54DnXR094654DKzKJv61i4OHOrxOitmjPA0uIjIYhHZIiKVInJzH9vTRORxd/tKESl1088TkdUistH9eU7IMa+651znPopiVyIz1o21kWJBZTbHmIkwz4KLiCQD9wDnA3OAK0VkTthu1wH7VXUmcDdwl5veCFykqicC1wB/DDvuKlWd5z7qo1YIc9QJzsE1FmsuYCPGTOR4WXNZCFSq6g5V7QYeA5aE7bMEeMB9/gTwCRERVV2rqnvc9E1Ahogk5lqzJqFsb2hjfFYqBdlj6+02LT+TNF+STQNjIsbL4DIVqAl5Xeum9bmPqvqBA0BB2D6fBdaoaldI2u/dJrHvST9r0IrI9SJSISIVDQ0NoymHOYpsq2tjZoLPhNyX5CThmMJsKm0CSxMhCd2hLyIn4DSVfSkk+Sq3uexj7nXtQTkAABl0SURBVOMLfR2rqveqarmqlhcWFkY/sybhqSqVDW3MTPAFwvpTVpRtNRcTMV4Gl91AccjraW5an/uIiA/IBZrc19OAp4GrVXV78ABV3e3+bAUewWl+M2bUmtq7aenoGZM1F3D6XXa3HKKj2+91VswY4GVwWQWUicgMEUkFlgLLw/ZZjtNhD3AZ8IqqqojkAc8BN6vq34M7i4hPRCa4z1OAC4F3o1wOc5QIfqtP9KWN+xMcMba9vt3jnJixwLPg4vah3ACsADYDy1R1k4jcJiIXu7v9DigQkUrgRiA4XPkGYCZwa9iQ4zRghYhsANbh1Hx+G7tSmbEs2B8x1kaKBR0eMdZgI8bM6Pm8vLiqPg88H5Z2a8jzTuBzfRx3O3B7P6c9NZJ5NCaosq6V7DQfk8ale52VqCgpyMKXJNbvYiIioTv0jYmlyoY2ji3Kpp8BiAkv1ZdESUGm3UhpIsKCizFDtK2ubczdmR+urCjHgouJCAsuxgzBgUM91Ld2jdn+lqCZRdlUN3fQ5e/1OismwVlwMWYIxuqcYuHKJmbTG1CqGju8zopJcBZcjBmC7fVje6RY0LGFNoGliQwLLsYMwbb6VtJ8SUzLz/Q6K1F1bGE2Ih8s5WzMSFlwMWYItta1cUxhNslJY3OkWFBGajLF+Zk2HNmMmgUXYwahqmyobeHEqeO8zkpMzJ06jvW1LV5nwyQ4Cy7GDKK6qYP9HT3Mn57vdVZiYn5xPrX7D9HQ2jX4zsb0w4KLMYNYV+N8i59XnOdxTmJj3nSnnMFyGzMSFlyMGcTaXfvJTE1m1sQcr7MSE3On5OJLEtbu2u91VkwCs+BizCDW1bRw0rTcMd+ZH5SRmszsyTlWczGjYsHFmAF09vTy3t6DR01/S9D84nzW17TQG1Cvs2ISlAUXYwawac9BenqV+UdJf0vQ/Ol5tHf32s2UZsQsuBgzgGC/Q7CT+2gRHLxg/S5mpDxdz8WMbYGAs+b86ur9dPX0UpiTTtG4NAqz0ygal0Zmavy//dbVtDA1L4OinLG5hkt/ZkzIIjcjhXU1LSxdON3r7AyqvctPfWsXDa1d1Ld2Un+wi+w0H6eU5HNsYdaYXSYhnsX/f7dJKO/uPsBrWxuoqGpmza4WDhzq6Xff048p4DsXHM/cqbkxzOHwrN3VctTVWgBEhHnFeazdFd+d+mt27eeO5zazurr/GlZ+ZgqnluRTXjqejx9XxHGTjo5Rf16z4GJG7UBHD8+s381j79Tw3t6DgDPB4/lzJx3+px6X7qOhrYv6g863y+rmDh56u5qLfvEmnz1lGjd98jgm5cZX7aC+tZPdLYf44kdLvc6KJ+ZPz+Nnf91GW5ef7LT4+qio3d/Bj17cwvL1eyjMSePG82YxLT+Dwpw0inLSKcxJY39HNxVVzVRU7Wd19X5e3lzPnS+8z8nFeSxdUMyFJ00mJz3F66KMWfH1jjEJQ1V5e0czj6/axQvv7qPLH+CEKeO4bckJXHDiZAqy0z50TEF2GrMnffD6ujNm8Mu/VfL7v1fx3Ia9fOmsY/jyWceSnpIcw5L0b537rX3+UVhzAaffRRU21LTwkZkTvM4O4DR/3fO3Su57cydJAl87ZyZfOutYsvoIfuOzUjm2MJsrFjjNevWtnTy7fi/LVtVwy1Mbue3Z97jgpMksXVDMqSX51nQWYRZczLA0tnXxxOpaHl9Vw87GdnLSfVxeXswVC4qH3byVm5HCLZ8+nqsWlXDXi+/z05e38c7OZu6/dkFcBJi1NS2kJAsnTInfZrtoOtypHyfBpb3Lz+d/t5K1u1r4zPyp3PSp45iSlzHk44ty0rnujBn8n4+Wsq6mhWUVNSxft4cnVtdSVpTN0oXT+cz8qeRnpUaxFEcPUfVuHLuILAZ+BiQD96nqnWHb04AHgVOBJuAKVa1yt90CXAf0Al9T1RVDOWdfysvLtaKiIlLFGnM6uv28sa2RZ9bt5i+b6vAHlAWl+SxdMJ1PnziZjNTIBIKn1tRy47L1fHLORH551Sn4kr0dzHjlvW/T3u1n+Q1neJoPL53zk1c5pjCb+64p9zQfXf5e/vmBCv5e2cgvrzqFxXMnR+S87V1+ntuwl0fe2cW6mhZSk5NYPHcSS+ZN4aMzJ8TFl5x4JiKrVbXPN4dnNRcRSQbuAc4DaoFVIrJcVd8L2e06YL+qzhSRpcBdwBUiMgdYCpwATAFeFpFZ7jGDnTNiVBVVUCBw+Ln7M+R5QBXFSXMODPk9JEGyCEkiiIAvSTz/UAWoO9jJy5vr+Ovmet6sbKTbHyA/M4VrP1LK0oXFzCyKfKfoZ06ZRmunn+8v38S3ntzIjy87iSSP7orvDTgzIV926jRPrh8v5k3P4/WtDaiqZ81GvQHl64+t441tjfz4spMiFlgAstJ8XL6gmMsXFLN570Eee2cXT6/dzfL1e0hPSeJjZYWcd/xEPj67iMKcDzf1xpKq0htQ/AEloEog+NkSCNnJ/ROJOE+DnyuC+9N9niTOoA0J7huFv62XzWILgUpV3QEgIo8BS4DQQLAE+IH7/AngF+L8FpYAj6lqF7BTRCrd8zGEc0bMb17fwZ0vvB/x8yYJpPqSSE1OItWXTEZqElmpPrLTfGSl+chKS2Zcegq5mSnkZvT/yElP6XfKElWlyx+gtdNPU3sXW/a1smVfK1vrWnl/Xyu1+w8BUDw+g88vKuHcOUUsKB1PSpQD3zUfKeXAoR7++6WtjMvwceuFczz5UNtW30p7d+9ROVIs1PziPJ5as5va/YcoHh/7hdJUlW8/tZEX3t3H9y6cw+fKi6N2reMnj+OHS+by7QuOZ+WOZl7eXMfL79Xx0nt1AEwfn8lxk3KYPSmHWROdx/isVHLSfaT5kvp9n/YGlIOHejjgPg52Oj9bOtzXIeltXb20d/mdR7efQ929dPkDdPsDdPcGiEZD0+2XzOXzp5VE/LxeBpepQE3I61pgUX/7qKpfRA4ABW7622HHTnWfD3ZOAETkeuB6gOnTRzaOv7wkn6+fW+Z8O+DIbwDBbwxJId8a3Os6P8GtzXzwLaQ34HwzCb6Ruv0BuvwBDnX7ae923nQtHd3U7vdzsNPPgUM9dPsDfeYtKCVZ3CDlPHxJSXR0+2nt9OMPm9rDlyQcU5jF/On5fP60Es6ZXURZUXbMP9z/5ZyZtHT0cP/fd5KXkcq/nlsW0+sDh4fgzi8+uqZ9CRec9mZtTUvMg4uq8p/Pb+bxihq+9okyrjtjRkyum+ZL5sxZhZw5q5AfXnwC7+09yKtbGnhvz0G21LXyyvv1H5oWJyVZyE7zkZnqwx9wg4H7f9zTO3BESPMlkZuRwriMFLLSfGSnJTM+K5PsNB/pKcmkuf+7wf/j5CRxHsFaifv5c7hhxI1AwdaTgNuSEjicHmxRcbafNC06fYpHbYe+qt4L3AtOn8tIzlFeOp7y0vERzddwdfb0Hv4GFPotKPhN6PC3HvfhDyhZaclkp/nITveRk+YjNzOVsqJsji3MJtXnfZOciPDdC47nYGcPd7+8lWn5GXw2xs1T63a1kJ+ZQknB2F7WeDDHTcohzZfEul0tXHzylJhe+6G3q/ntGzu59iOl/JsHXzDAeS+eMCX3iEEdnT29bG9oo7K+jYOHejjY6aety09bp1PbSEn64Mtcmi+JNF8y4zJ8TgDpo8VhrPbreBlcdgOhddxpblpf+9SKiA/IxenYH+jYwc45pqSnJDMpNznu7hEZraQk4c7PnMiu5g5+sHwTpx9bMKyRQaO1tmY/84rzjvrhqSnJSZw0LZe1NbGdBmZHQxt3PL+Zs48r9KxptD/pKckfCjjmw7z8mroKKBORGSKSitNBvzxsn+XANe7zy4BX1KnzLQeWikiaiMwAyoB3hnhOkyB8yUn85LKT6VXlW09uIFYjG1s7e9hW38a8o7xJLGhecR6b9hyky98bk+v1BpSb/nc9ab5k7vqsd4M6zOh4FlxU1Q/cAKwANgPLVHWTiNwmIhe7u/0OKHA77G8EbnaP3QQsw+mofxH4qqr29nfOWJbLRNb0gky+/enjeWNbIw+v3BWTa26oPYDq0XvzZLj50/Pp9gfYvLc1Jtf77Rs7WLOrhduWnMDEcWOrRn408bTPRVWfB54PS7s15Hkn8Ll+jr0DuGMo5zSJ7apF01mxaR//+fxmziwrZHqU+0GCi2SdfJRNs9+f4M2U63btj/pSz1vrWvnvv2xl8QmTYt7HYyLL+95bYwYhItz12ZNIFuGmJ9YTiPICVq9vbWDWxGxyM2zeKYDJuelMzcvg9W2NUb1OT2+AG5etIyfdx+2Xzo2rfhYzfBZcTEKYkpfBrRfN4Z2dzfz+H1VRu87ulkOs3NnMRSfZt+YgEeHCkybz+tYGmtq6onadX/5tO+/uPsgdl85lQh9z05nEYsHFJIzLTp3GJ2YX8aMX32d7Q3RWSHxmnTO48JL5UwfZ8+hyyfyp+APKcxv3RuX8m/Yc4H9e2caSeVMiege+8Y4FF5MwRIT/+syJpPmS+M7TGyM+ekxVeXrNbhaU5ntyN3o8O37yOGZPyuGpNZEf2d8bUG55aiN5man88OITIn5+4w0LLiahFI1L5+bzj+ftHc08sbo2oufetOcg2+rbrNbSj0vnT2VdTQs7G9sjet4H36piQ+0Bbr1oDnmZNiPxWGHBxSScpQuKKS/J547nN0e0D+DptbtJTU7ighOtWaYvF8+bggj8aW3kai97Wg7xkxVbOGtWIRedZL/3scSCi0k4SUlO81h7l587ntsckXP6ewMsX7+Hj88utG/P/Zicm8FHji3gT+t2R6xJ8gfLN9Gryu2X2OiwscaCi0lIZRNz+NKZx/LU2t28GYEhsv/Y3kRDaxeXWpPYgC6ZN5Xqpg7WuBN7jsaKTfv4y3t1fP3cWdbHNQZZcDEJ64ZzZlJakMl3/7SRzp7RTU3y9NrdjEv38fHZRRHK3di0eO4k0nxJo24aa+vy8/1nNjF7Uk7MZjs2sWXBxSSs9JRk7rj0RKqaOvjFK5UjPk97l58X393HBSdNIc03NmeojZSc9BQ+ecIknt2wZ9DlHgbykxVbqGvt5L8+c2LU1wgy3rC/qkloH505gc/Mn8qvX9vOxtoDIzrHX97bx6GeXmsSG6JL50+hpaOH17Y2jOj4VVXNPPBWFV84reTwejFm7LHgYhLedy+cQ1FOGl9+aDXN7d3DPv7ptXuYmpdBeYl90A3Fx8oKKchKHVHTWN3BTr7y8BpKxmfyjU8dF4XcmXhhwcUkvPFZqfzq86fS0NbFvzy6Bn/v0Jtr6ls7eXNbA5fOn2pTuw9RSnISF508hZc213Gws2fIx3X7A/y/h1bT3uXn3qvLyUm3udvGMgsuZkw4uTiP25fM5e+VTfx4xZYhH/fQW9UEFC6Zb3OJDccl86fS7Q/w6DCWQfjhs5tYs6uFH192MrMm5kQxdyYeWHAxY8blC4r5/GnT+c3rO/jzhj2D7v/Ht6r4+SuVXHDSZGYW2YfdcJw8LZezjyvkzhff58khzJTw+KpdPLxyF18+61gusJsljwoWXMyYcuuFJ3BqST7ffGIDW/b1v7jVIyt38b1nNnHu8UXcffm8GOZwbBARfv35Uzn9mAJuemL9gP0v62pa+N6fNvGxsgnWz3IUseBixpRUXxK/vOoUstJ8/PODq1i+fs+H7oF5fNUuvv30Rs6ZXcQ9V51Cqs/+DUYiPSWZ312zgEUzxnPjsnUsX39kbfFQdy9Pranly39cTdG4NH6+dD7J1q911JBYrUsez8rLy7WiosLrbJgIWrNrP//yyFp2txxiXLqPS+ZP5fLyYjbvPcg3n9zAmWWF/OYLp5KeYve1jFZHt59r71/F6l37+dnSeUzNy2BZRS3Prt9DW5ef0oJMfnnVqcyZMs7rrJoIE5HVqlre5zYLLhZcxqpAQPnH9iaWVdTw4qZ9h2/6+1jZBH57dbkFlghq7/Jzzf3vUFG9H4D0lCQ+feJkLi8vZtGM8TZv2BgVd8FFRMYDjwOlQBVwuaru72O/a4Dvui9vV9UHRCQT+F/gWKAXeFZVb3b3vxb4MRBsAP6Fqt43WH4suIx9Bzp6WL5+N1VNHXzjU8dZYImCti4/P1mxheMm5XDhSZNtqPFRIB6Dy4+AZlW9U0RuBvJV9Vth+4wHKoByQIHVwKlAF7BIVf8mIqnAX4H/VNUX3OBSrqo3DCc/FlyMMWb4BgouXvVkLgEecJ8/AFzSxz6fAl5S1Wa3VvMSsFhVO1T1bwCq2g2sAabFIM/GGGOGyKvgMlFVg4tx7wMm9rHPVKAm5HWtm3aYiOQBF+HUXoI+KyIbROQJESmOYJ6NMcYMkS9aJxaRl4FJfWz6TugLVVURGXbbnIj4gEeBn6vqDjf5WeBRVe0SkS/h1IrO6ef464HrAaZPnz7cyxtjjBlA1IKLqp7b3zYRqRORyaq6V0QmA/V97LYbODvk9TTg1ZDX9wLbVPWnIddsCtl+H/CjAfJ3r3sOysvLbcicMcZEkFfNYsuBa9zn1wDP9LHPCuCTIpIvIvnAJ900ROR2IBf4eugBbqAKuhiIzBq4xhhjhsWr4HIncJ6IbAPOdV8jIuUich+AqjYD/wGsch+3qWqziEzDaVqbA6wRkXUi8s/ueb8mIptEZD3wNeDaWBbKGGOMw26ixIYiG2PMSMTjUGRjjDFjmNVcABFpAKpHePgEoDGC2fGClSF+jIVyWBniQyzKUKKqhX1tsOAySiJS0V+1MFFYGeLHWCiHlSE+eF0GaxYzxhgTcRZcjDHGRJwFl9G71+sMRICVIX6MhXJYGeKDp2WwPhdjjDERZzUXY4wxEWfBxRhjTMRZcBkFEVksIltEpNJd9Czuicj9IlIvIu+GpI0XkZdEZJv7M9/LPA5GRIpF5G8i8p473c+/uukJUw4RSReRd0RkvVuGH7rpM0RkpfueetxdEC+uiUiyiKwVkT+7rxOqDCJSJSIb3amkKty0hHkvgbP8iLvMyPsisllETve6DBZcRkhEkoF7gPNx5jm7UkTmeJurIfkDsDgs7Wbgr6pahrM2TrwHSj/w76o6BzgN+Kr7u0+kcnQB56jqycA8YLGInAbcBdytqjOB/cB1HuZxqP6VIyeJTcQyfFxV54XcF5JI7yWAnwEvqups4GScv4e3ZVBVe4zgAZwOrAh5fQtwi9f5GmLeS4F3Q15vASa7zycDW7zO4zDL8wxwXqKWA8jEWVF1Ec4d1T43/Yj3WDw+cJbC+CvOukl/BiQBy1AFTAhLS5j3Es4M8TtxB2jFSxms5jJyg66UmUCGsjJoXBKRUmA+sJIEK4fbnLQOZz2jl4DtQIuq+t1dEuE99VPgm0DAfV1A4pVBgb+IyGp3EUFIrPfSDKAB+L3bPHmfiGThcRksuJgjqPM1JyHGp4tINvAk8HVVPRi6LRHKoaq9qjoP59v/QmC2x1kaFhG5EKhX1dVe52WUzlDVU3CauL8qImeGbkyA95IPOAX4larOB9oJawLzogwWXEZuN1Ac8nqam5aI6oILrQ2wMmhcEZEUnMDysKo+5SYnXDkAVLUF+BtOE1Keu4Q3xP976qPAxSJSBTyG0zT2MxKrDKjqbvdnPfA0TqBPpPdSLVCrqivd10/gBBtPy2DBZeRWAWXuyJhUYCnOCpuJaCgrg8YNERHgd8BmVf3vkE0JUw4RKRSRPPd5Bk6f0WacIHOZu1tcl0FVb1HVaapaivP+f0VVryKByiAiWSKSE3yOs+LtuyTQe0lV9wE1InKcm/QJ4D08LoPdoT8KIvJpnDbnZOB+Vb3D4ywNSkQeBc7GmY67Dvg+8CdgGTAdZ+mBy9VZCTQuicgZwBvARj5o6/82Tr9LQpRDRE4CHsB57yQBy1T1NhE5BqcWMB5YC3xeVbu8y+nQiMjZwE2qemEilcHN69PuSx/wiKreISIFJMh7CUBE5gH3AanADuCLuO8rPCqDBRdjjDERZ81ixhhjIs6CizHGmIiz4GKMMSbiLLgYY4yJOAsuxhhjIs6CizFRIiLfcWc83uDOuLsoitd6VUTKB9/TmNjwDb6LMWa4ROR04ELgFFXtEpEJOPcgGHNUsJqLMdExGWgM3jyoqo2qukdEbhWRVSLyrojc6842EKx53C0iFe56HAtE5Cl3LY7b3X1K3fU6Hnb3eUJEMsMvLCKfFJG3RGSNiPyvOwebMTFlwcWY6PgLUCwiW0XklyJylpv+C1VdoKpzgQyc2k1QtzrrifwaZ6qOrwJzgWvdO8YBjgN+qarHAweBr4Re1K0hfRc4152MsQK4MTpFNKZ/FlyMiQJVbQNOBa7HmQ79cRG5Fvi4u0rjRpyJHk8IOSw4N91GYJOq7nVrPjv4YJLUGlX9u/v8IeCMsEufhrN43d/d6fyvAUoiWjhjhsD6XIyJElXtBV4FXnWDyZeAk4ByVa0RkR8A6SGHBOffCoQ8D74O/q+Gz9cU/lqAl1T1ylEXwJhRsJqLMVEgIseJSFlI0jyclQEBGt1+kMs+fOSgpruDBQD+CXgzbPvbwEdFZKabjywRmTWC6xgzKlZzMSY6soH/cafV9wOVOE1kLThTuu/DWbZhuLbgLGh1P8606r8K3aiqDW7z26MikuYmfxfYOpJCGDNSNiuyMQnCXdL5z+5gAGPimjWLGWOMiTiruRhjjIk4q7kYY4yJOAsuxhhjIs6CizHGmIiz4GKMMSbiLLgYY4yJuP8f4qhAidPtIUcAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "cutoff = 0.15\n", + "beta = 9\n", + "ntaps = 63\n", + "N = 4\n", + "\n", + "b = sig.firwin(ntaps, cutoff, window=('kaiser', beta))\n", + "w, h = sig.freqz(b)\n", + "\n", + "plt.plot(b)\n", + "plt.title('Resulting PQMF Window Function')\n", + "plt.xlabel('Sample')\n", + "plt.ylabel('Value')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAEWCAYAAADl19mgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nOydd3hVVdb/PyudhN6r9F4SEBTbCIoDCoqAIkrJ1XF8HXVmnFffmZ8z48So4ziOfazojBQLomLFLiAWUBAIvXcIvbeEJOv3xz7n5ubmtoQkN4n78zznufecs84+617C+d6999priapisVgsFkt1IibaDlgsFovFUtZYcbNYLBZLtcOKm8VisViqHVbcLBaLxVLtsOJmsVgslmqHFTeLxWKxVDusuFlCIiIviMi9ZW0b5HoVkQ5BzjURkbkiclREHhORP4vIy865Ns61caW9dwS+Fbl/ed3HYrGUDeX2MLBUfkRkM9AEyAPygZXAFGCiqhYAqOqtkbbnaysiA4BXVbVlGbl7C7APqK1hFmeKyBzn3i+X0b1LdH+LxRJ9bM/NcqWq1gJaAw8DfwL+E12XAtIaWFkRwiIisSW9f3n2GisjQb4ji6XyoKp2+5luwGZgkN+xc4ACoIezPwl40Of8H4FsYCdwM6BAB19bIAU46bRzzNmaO23PAw45bTwDJPi07W3Lz6dJwGkg12lrEHAfpncG0Ma5Ng74O6YXesqxfcax6QJ8ARwA1gCj/dp/HvgYOB7gOwl2/7eBV4EjzndRB/PDIBvY4XwXsU4bscCjmN7fRuB21+dA/xa+n8/Z7w9873x3WcAAn3NzgAeA74CjwOdAQ5/zF/pcuw3wAP2A3a5/jt1IICvI30qx78j5N30H2AtsAn7n93e00PludgOP+/1b3YL5G8oG7va5LhF40jm303mf6JwbAGwH7gL2ONfe6HPtFZjRh6PO9+/b7jBgifMdfA/0ivb/P7uV7xZ1B+wWxX/8AOLmHN8K/MZ5PwlH3IAhwC6gO5DsPNiLiZvzfgCw3a/ds52HdJzzkFsF3OlzPqC4+bft7Hsf/j4PTFco5gA3+9imOA/1G51798aITDeftg8DF2BGM5IivP9p4GrnmhrAu8CLzv0aAz8C/+PY3wqsBloB9YHZRChuQAtgv/PwjgEuc/Yb+XzeDUAnx485wMPOudaYh/31QDzQAEhzzq0ELve557vAXSG+f9/vKBn4CfgbkAC0w4j2YMd+HjDeeV8T6O/3b/WG8z31xIjjIOf8/cB85/trhBGiB3z+pvIcm3jn+zgB1HPOZwMXOe/rAX2c970xYngu5kdGuvN9J0b7/6Ddym+zw5KWQOzEPID9GQ28oqorVPUE5gEcMar6k6rOV9U8Vd2MEYKLz9TZCBgGbFbVV5x7L8b0OK71sXlfVb9T1QJVPRVhu/NU9T0185O1MQ/bO1X1uKruAZ4Axji2o4EnVXWbqh4A/lEC/8cBH6vqx45/X2B6RVf42LyiqmtV9SQwHUhzjt8AfKmqb6jqaVXdr6pLnHOTnbYRkfrAYOD1EH54vyOMKDVS1ftVNVdVNwIv+Xze00AHEWmoqsdUdb5fW5nO97QMeAUjvgBjgftVdY+q7gUygfE+1512zp9W1Y8xPenOPue6iUhtVT2oqouc47cAL6rqD6qar6qTgRzMDy1LNcWKmyUQLTDDd/40x/SAXLYFsAmKiHQSkY9EZJeIHAEeAhqW3s2IaQ2cKyKH3A3zEG3qY1OizxLgmtaY3kS2zz1exPRAoPh3t6UE92kNXOvn/4VAMx+bXT7vT2B6S2B6ihuCtPsqcKWIpGDE9xtVzQ7hh//nbe7n058xAUoAv8L0JFeLyAIRGRairS2Y7wfndUuQcwD7VTXPZ9/3s47CCP4WEflaRM7z8fUuP19b+bVrqWb8rCbBLeERkX4Ycfs2wOlswDf6sVWIpgIFXjwPLAauV9WjInIncE1pfS3BvbcBX6vqZSW4pqT32YbpDTT0e/i6ZFP0+zrL7/xxzFCfi7/wTlXVX5fCx22Y+a9iqOoOEZmHmWsbj/n3CYX/592kqh2DtL0OuF5EYpz23xaRBj4mrTDDtGC+i53O+50YMVoR4Fxo51QXAMNFJB64A9ODbeX4+ndV/Xsk7ViqB7bnZgFARGo7v66nYeZ6lgUwmw7cKCJdRSQZCLWmbTfQQETq+ByrhQkwOCYiXYDflJH7ge7dzmf/I6CTiIwXkXhn6yciXcvqhk6P53PgMee7jBGR9iLiDrtOB34nIi1FpB7w//yaWAKMcXzrS1HRd3tYg0UkVkSSRGSAiESyzOI1YJCIjBaROBFpICJpPuenYIKEegIzSvCRfwSOisifRKSG41cP58cRIjJORBo5Q5iHnGsKfK6/V0SSRaQ7Zi70Tef4G8BfRaSRiDTEzOm9Gs4ZEUkQkbEiUkdVT2P+ztz7vQTcKiLniiFFRIaKSK0SfF5LFcOKm+VDETmK+XX7F+BxzMOmGKr6CfA0JhhiPWbiH0yPxd92NeZBtdEZCmoO3I2ZAzqKeeC86X9dGfEUcI2IHBSRp1X1KPBLzHzQTswQ3j8xkXllyQRMcMVK4CAmmtIdOnwJ+AwT6biI4kJyL9DeuS4Tn7kvVd0GDMcM++3F/Fv9HxH8/1XVrZihurswQ81LgFQfk3cxPaV3nXnUiFDVfMxcZhomUnIf8DImYhRM8NEKETmG+fcY48wHunyN+Rv6CnhUVT93jj+ImU9cCizDfFcPRujWeGCzM+R9K2boGVVdCPwaE5170LmvJ9LPaqmaiKpdj2opHU7PZzkm6izQUJwlCCLSBiMK8dH+7kRkAyaq88sKuFcbKsnntlRvbM/NUiJEZISIJDpDa/8EPrQPqaqLiIzCzKXNirYvFktZYsXNUlL+B7NmaANmsXR5zZtZyhknTdnzwO3O3JjFUm2ww5IWi8ViqXbYnpvFYrFYqh3Vfp1bTEyM1qhRI9puWCwWS5XhxIkTqqpVuvNT7cWtRo0aHD9+PNpuWCwWS5VBRE6Gt6rcVGlltlgsFoslEFbcLBaLxVLtsOJmsVgslmqHFTeLxWKxVDusuFksFoul2lHlxE1EhojIGhFZLyL+mdUtFovFUgFU9mdxlRI3EYkFngUuB7ph6kV1i65XFovF8vOiKjyLq9o6t3OA9U5Je0RkGqYUyMqyvMnJ0ycZ8Y/naJTXm3ZyCSKU21azJtSrV7i1aAF165blp7FYLJYyp0KexWdCVRO3FhQtT78dONffSERuAW4BaNGiBXPmzCnRTfI1n+95iLMSupBWo/SdW1WzheLkSdi7t+ixuDhITIQaNYz41axp9i0Wi6WCiBORhT77E1V1os9+RM/iaFLVxC0inH+EiQApKSk6YMCAErfxP6dv4skfnmT8rd1olNzYK1RlvR07BgcPwoED5nXbNli1CtauhcWL4ZBTw7hLFxg5EiZMgM6dy/DLslgsluLkqWrfaDtxJlQ1cdsBtPLZb+kcK3PS09J5dN6jvL7sde7sfyci5XEXaNIk+LmCAlizBmbNgnffhX/+Ex56CAYPhrvvhkGDyscni8ViCUOFPYtLS5UqeSMiccBa4FLMF7kAuEFVVwS7JiUlRUubW7LfS/04nX+aJbcuKdX1Zc3u3TBxIjz/PGRnG5F79FHo0SPanlksluqEiJxQ1ZQQ50v8LK5oqlS0pFPx+Q7gM2AVML08v8z01HSydmexZFflELcmTeDee2HTJnj8cfjhB0hNhb/+FXJzo+2dxWL5uVDRz+LSUKV6bqXhTHpu+0/sp9ljzbi93+08MeSJMvbszNm/3wxPTpoEffrAjBnQunW0vbJYLFWdcD23qkCV6rlVNA2SG3BV56t4bdlrnM4/HW13itGgAbzyipmP27AB+vaFuXOj7ZXFYrFEHytuYUhPTWfvib18sv6TaLsSlKuvhh9/hIYN4bLL4IMPou2RxWKxRBcrbmEY0mEIjVMaM2nJpGi7EpJOneC77yAtzSwZePPNaHtksVgs0cOKWxjiY+MZ13McH639iH0n9kXbnZDUrw9ffgnnnw/jxsGnn0bbI4vFYokOVtwiID0tndMFp3lj2RvRdiUstWrBhx9C9+4wahQsWBBtjywWi6XisdGSEdLnxT6ICD/d8lMZeFX+7NoF/ftDXh789FPoxeIWi8Xii42W/BnhSfOwKHsRy3Yvi7YrEdG0Kbz3nknrde21cLryBXtaLBZLuWHFLUKu73E9cTFxTM6aHG1XIiYtDV5+Gb75Bh58MNreWCwWS8VhxS1CGqU0YlinYby69FXyCvKi7U7E3HADpKcbcZs3L9reWCwWS8Vgxa0EeFI97D6+m8/WfxZtV0rE00+bzCXjxsGJE9H2xmKxWMofK24l4PKOl9MwuSGTsiZF25USUbu2yWSycSM88EC0vbFYLJbyx4pbCUiITWBsz7F8sOYDDpw8EG13SsTFF4PHY6oILF8ebW8sFoulfLHiVkI8aR5y83OZtnxatF0pMf/6F9SpA7ffHr5CuMVisVRlrLiVkLSmafRq0qvSp+MKRMOGZlhy7lyz0NtisViqK1bcSoEn1cOCnQtYuXdltF0pMTffDJ07w5/+ZBZ4WywWS3XEilspGNtrrFnztqTqrHlziY+Hhx+G1avhP/+JtjcWi8VSPlQ6cRORf4nIahFZKiLvikhdn3P3iMh6EVkjIoOj5WPjlMZc3uFypi6dWqXWvLkMH25Scz30kM1cYrFYqieVTtyAL4AeqtoLWAvcAyAi3YAxQHdgCPCciMRGy0lPmofsY9l8seGLaLlQakTg3nth61aYOjXa3lgsFkvZU+nETVU/V1W3OzQfaOm8Hw5MU9UcVd0ErAfOiYaPAEM7DqV+jfpVKh2XL5dfDn36wD/+YefeLBZL9aPSiZsfNwFuCewWwDafc9udY8UQkVtEZKGILMwrpyd3YlwiN/S4gfdWv8fBkwfL5R7liQj89a+wfj28/Xa0vbFYLJayJSriJiJfisjyANtwH5u/AHnAayVtX1UnqmpfVe0bFxdXlq4XwZPmISc/hzdXVM2y18OHQ4cO8NRT0fbEYrFYypaoiJuqDlLVHgG29wFExAMMA8ZqYcG5HUArn2ZaOseiRp9mfejeqHuVHZqMiYHf/hbmz4cff4y2NxaLxVJ2VLphSREZAvwRuEpVfdP8fgCMEZFEEWkLdASi+kgWETxpHuZvn8/qfauj6Uqp8XhM9e6nn462JxaLxVJ2VDpxA54BagFfiMgSEXkBQFVXANOBlcCnwO2qmh89Nw1je44lVmKr5Jo3MEmVb7oJpk+H3buj7Y3FYrGUDaLVPMlgSkqKHj9+vFzvMfT1oWTtymLLnVuIjYna6oRSs3o1dO1qck/efXe0vbFYLNFGRE6oakq0/TgTKmPPrcrhSfWw4+gOvtr0VbRdKRVdusB558F//2sTKlssluqBFbcy4MrOV1IvqV6VDSwBMzS5ahX88EO0PbFYLJYzx4pbGZAUl8SYHmOYsWoGh08djrY7pWL0aEhONkVNLRaLpapjxa2M8KR5OJV3iukrpkfblVJRuzZcey1MmwY5OdH2xmKxWM4MK25lRL/m/ejasGuVHpocMwaOHIHPPou2JxaLxXJmWHErI0SE9NR0vtv2Hev2r4u2O6Xi0kuhfn2zLMBisViqMlbcypBxvcYRIzFVtvcWHw8jR8L778PJk9H2xmKxWEqPFbcypEXtFvyy/S+ZkjWFAi2ItjulYvRoOHbMDk1aLJaqjRW3MiY9NZ1tR7Yxe9PsaLtSKgYOhIYN7dCkxWIpHSJyn4jscDJMLRGRK6LhhxW3MmZ45+HUSazDpKxJ0XalVMTFwbBh8Mknts6bxWIpNU+oapqzfRwNB6y4lTE14mswpscY3ln5DkdyjkTbnVIxdCgcOgTffx9tTywWi6V0lF+xs0pC/fr1mTNnToXes2d+T07mneTBGQ9yRbOo9MjPiPr14bHHYNs2qOCvzmKxVA7iRGShz/5EVZ1YguvvEJEJwELgLlWt8IrONnFyOaCqdHm2C01SmjD3xrkVeu+y4tJLTZWA5cuj7YnFYqlowiVOFpEvgaYBTv0FmA/sAxR4AGimqjeVi6MhsMOS5YCI4En18M3Wb9hwYEO03SkVw4bBihWweXO0PbFYLJWNUAWnVXW3quaragHwEnBONHy04lZOjE8djyBMyZoSbVdKxdCh5nXmzOj6YbFYqhYi0sxndwQQlfEfK27lRMvaLRnUbhCTsyZXyTVvnTpB69Ywa1a0PbFYLFWMR0RkmYgsBQYCf4iGE5VW3ETkLhFREWno7IuIPC0i60VkqYj0ibaP4UhPTWfL4S3M3VI1590GDjQBJQVVT5stFkuUUNXxqtpTVXup6lWqmh0NPyqluIlIK+CXwFafw5cDHZ3tFuD5KLhWIkZ0HUGthFpMWjIp2q6UioED4cABWLYs2p5YLBZLyaiU4gY8AfwRE23jMhyYoob5QF2/sd1KR3J8Mtd1v463V77Nsdxj0XanxAwcaF7t0KTFYqlqVDpxE5HhwA5VzfI71QLY5rO/3TlWqUlPS+f46eO8s/KdaLtSYlq1gvbtYXbVzCRmsVh+xkRF3ETkSxFZHmAbDvwZ+NsZtn+LiCwUkYV5Uc4hdUGrC2hfr32VTcc1cCDMnQv5+dH2xGKxWCInKuIWbI0EsBFoC2SJyGagJbBIRJoCO4BWPs20dI4Fan+iqvZV1b5xcdFNwiIieNI8zNk8h82HNkfVl9IwcCAcPgyLF0fbE4vFYomcSjUsqarLVLWxqrZR1TaYocc+qroL+ACY4ERN9gcORysKp6SM7zUeoEquebvoIvM6b150/bBYLJaSUKnELQwfY3p26zGr3m+LrjuR07puay5pewmTsyZT1dKdtWoFLVpYcbNYLFWLSi1uTg9un/NeVfV2VW3vrKFYGO76yoQn1cPGgxv5duu30XalxPTvD/PnR9sLi8ViiZxKLW7ViZFdR1IzoWaVXPN23nmwaZNJpGyxWCxVAStuFURKQgrXdruW6Sunczy3YqsUnCnnnWde7dCkxWKpKlhxq0A8aR6O5R7j3dXvRtuVEtGnj6nQ/eOP0fbEYrFYIsOKWwVy4VkX0rZu2yo3NJmUBN262eUAFoul6mDFrQKJkRjSU9OZtWkWWw9vDX9BJaJPH1i0CKpYsKfFYvmZYsWtgpmQOgFFmZo1NdqulIg+fWDPHsiuEisLLRbLzx0rbhVM23ptubj1xUzKmlSl1rz1cQoMLVoUXT8sFoslEqKbm+pniifNw43v38j3277ngrMuiLY7EZGaCiJG3IYNi7Y3FoulOiOZch4wDrgIaAacxFT0ngm8qhl6OGwbwXoPInwQgQ8HVPFE6nA0SElJ0ePHK1fo/bHcYzR9tCk39LyBiVdOjLY7EdO5M3TvDjNmRNsTi8VSnojICVVNicq9M+UTYCfwPrAQ2AMkAZ0wlb2vBB7XDA2pUaF6bl2Bm0P5ADxbAp8tDjUTajKq2yjeXPEmTw55kuT45Gi7FBE9esDy5dH2wmKxVHPGa4bJTOXDMWCRsz0mmdIwXCOhxO0vqnwd6mIRMsO6aQmIJ9XDlKwpvLf6PW7oeUO03YmIbt3gvfcgJwcSE6PtjcViqY64wiaZkgKc1AwtkEzpBHQBPtEMPR1A/IoRNKBElelhnYjAxhKYi9tcTOs6rZmcNTnarkRM9+5QUABr1kTbE4vF8jNgLpAkmdIC+BwYD0yK9OKg4iZCQxEyRPidCDVFeF6E5SK8L0KHM3b7Z06MxDAhdQJfbPiC7Ue2R9udiOje3byuWBFdPywWy88C0Qw9AYwEntMMvRboHunFoZYCvA4kAh2BHzHlZq4BPgJeLrW7Fi/pqelVas1bp04QGwsrV0bbE4vF8jNAnKjJsZgoSYDYSC8OJW5NVPkz8Dugpir/UmW1Ki8BdUvtrsVL+/rtufCsC6tMnbfEROjY0fbcLBZLhfB74B7gXc3QFZIp7YDZkV4cKqAkH0AVFcF/8q6gxG5aAuJJ9XDzhzfzw44f6N+yf7TdCUu3blbcLBZL+aMZOhcz7+bub8R0tiIilLi1c9a6ic97nP22pfA1YkTkt8DtGIGdqap/dI7fA/zKOf47Vf2sPP2oCK7tfi2//eS3TFoyqUqIW4cO8NFHkJ9vhigtFoulPHAiJO8G2uCjVZqhl0RyfShxG+7z/lG/c/77ZYaIDHTunaqqOSLS2DneDRiDmVBsDnwpIp1UNb+8fKkIaifWZmTXkUxbPo0nhzxJUlxStF0KSfv2kJsLO3bAWWdF2xuLxVKNeQt4ARPjUeLnfFBxC7fGrRz5DfCwquYYP3SPc3w4MM05vklE1gPnAFW+hKYnzcNry17j/dXvc12P66LtTkjatzevGzZYcbNYLOVKnmbo86W9OKi4ibAMCBrloEqv0t40DJ2Ai0Tk78Ap4G5VXQC0AOb72G13jlV5BrYZSKvarZicNbnSi1sHZxHI+vUwcGB0fbFYLNWaDyVTbgPeBXLcg5qhByK5ONSwpJse93bn1Y1XH0cI0YsEEfkSaBrg1F8cn+oD/YF+wHQRaVfC9m8BbgFISEg4E1crhNiYWMb3Gs/D3z3MzqM7aV6rebRdCkrLlhAfb3puFovFUo6kO6//53NMgYj0IGjiZK+BsFiV3n7HFqnSpyReRoqIfAr8U1VnO/sbMEJ3M4Cq/sM5/hlwn6qGHJasjImTA7F2/1o6P9OZfw76J3+84I/RdicknTtDr17w1lvR9sRisZQH0UycXFZEUvJGRLhAle/MDudTvnXg3sNkfp4tIp2ABGAf8AHwuog8jgkocReXVws6NejE+a3OZ3LWZP7v/P9DRKLtUlDatzfDkhaLxVKeSKb0ALphqgIAoBk6JZJrIxGpXwHPibBZhM3Ac8BNpfAzUv4LtBOR5cA0IF0NK4DpwErgU+D2qh4p6U96ajor965k4c6F0XYlJB06mGHJKrDu3GKxVFEkUzKAfzvbQOAR4KpIrw8rbqr8pEoqmE2VNFXKrR6zquaq6jhV7aGqfVR1ls+5v6tqe1XtrKqflJcP0WJ099EkxSUxacmkaLsSkrZt4ehROHgw2p5YLJZqzDXApcAuzdAbMRpUJ9KLQyVOLlJvWZXDqhwOZWM5M+om1WVElxG8sfwNcvJywl8QJVq1Mq/btkXXD4vFUvkQkWtFZIWIFIhIX79z94jIehFZIyKDwzR1UjO0AMiTTKmNKVraKlI/QvXc/iVCbxH6BNuAhyK9kSUy0lPTOXjqIB+u/TDargTFipvFYgnBckwm/7m+B/0ScQwBnhORUHmOFkqm1AVeAn7CFCqNeF1zqICS3cDjYa5fF+mNLJExqN0gmtdqzqQlk7im2zXRdicgrrht3RpdPywWS+VDVVcBgYLiSpSIQzP0NuftC5IpnwK1NUOXRupHqAwlAyJtpDJTv3595syZE203SsSAugOYtm4aMz6fQf2E+tF2JyCPPQZ16kAV+2otFktkxImIb2TbRFWdeIZtRpSIQzKli2boasmUYsvNJFP6aIZGFPMRyVKAKs2BAwcYMGBAtN0oEU17NOX1Z19nU81NjDx/ZLTdCYjHAxddBFOrRik6i8VSMvJUtW+wk6EScajq+2d477uAXwOPBTinwBknTrZEiS4Nu3Bui3OZlDWJ/z3vfyvlmrdmzWDXrmh7YbFYooGqDirFZTsoGhDS0jlWtO0M/bXzekYJ/qy4VVI8aR5+M/M3LN61mD7NyiUZzBnRtKldyG2xWEpERIk4JFNCDldphs6I5GZh17mJkCzCvSK85Ox3tEsAyp/rul9HYmxipV3z1rQpZGdH2wuLxVLZEJERIrIdOA+Y6aRKpASJOK50tl8B/wHGOtvLlCCBSCQZSl7BZGQ+z9nfATwY6Q0spaNejXoM7zKc15e9Tm5+brTdKUazZrB/v6ntZrFYLC6q+q6qtlTVRFVtoqqDfc6FTcShGXqjs2g7HuimGTpKM3QUZglBfKR+RCJu7VV5BDhtnOMEphq3pZzxpHrYf3I/M9fOjLYrxWjqTCXv3h1dPywWS7WllWao7/jQbiDiKpKRzLnlilADp8yNCO3xqa1jKT8ua38ZTWs2ZVLWJEZ0HRFtd4rgituuXYXr3iwWi6UM+Uoy5TPgDWf/OuDLSC+OpOeWgRkfbSXCa8BXQOWuyVJNiIuJY3yv8Xy87mP2HN8T/oIKpGFD87p/f3T9sFgs1RPN0DuAF8Gb23iiZuhvI70+bD03ABEaYGqqCTBflX2lc7fiqSr13IKxYs8KejzfgycGP8Gd/e+Mtjte1qyBLl3g1Vdh7Nhoe2OxWL7b+h39W/YnNiZURqvIqA713EIlTvbNIdkayAZ2Amc5xywVQPfG3enbvG+li5ps0MC82p6bxVK+LNy5kFmbZoW02XBgAxe+ciEv/vRiBXlV/kim9JdMWSCZckwyJVcyJV8y5Uik14calnzM2Z4FfgAmYhJY/uAcs1QQnlQPWbuzWLJrSbRd8VKvnnm14maxlI5jucf4zUe/Ye/xvSHt7v/6fq6Zfk3ISiGHc0zBlleWvFKmPkaZZ4DrMTmMawA3UwLtCSpuqgxUZSCmx9ZHlb6qnA30JsCqckv5MabHGOJj4pm8ZHK0XfESGwt161pxs1gC8eDcB/nb7L+FtFmUvYgXfnqB5xc+H9LuZN7JsJVCXOFbuHMhK/asKLnDlRTN0PVArGZovmboK5hqAhERSUBJZ1WWeW+mLAe6ltxNS2lpkNyAqzpfxWvLXuN0/ulou+OlQQMrbhZLIGaum8kj3z3CoVOHgtq461cnZ00mVOyDaxdqasJ3LezkrMrzI/gMOSGZkgAskUx5RDLlD0SmWRCh4VIRXhZhgLO9BERcdqCkiEiaiMwXkSUislBEznGOi4g87RS6WypSPGN0dcaT5mHvib18sr7yFCC34mb5ObJy70p+2P5DSJvc/Fxy8nN4c/mbIW0ANh7cyLdbvw1r9+n6T9l1LHBCV9emSUoTpi6dSl5BXkj/qgjjMRp1B3Ack5dyVKQXRyJuNwIrgN8720rnWHnxCJCpqmnA35x9gMsxucg6ArcAofvy1YzB7QfTOKVxpQosadAADhyItn0ECy4AACAASURBVBcWS9lwKu8U93x5DwdPHgxp99dZf2Xo60NDZg7y7ZWFs4HwvbJODTqRr/m8tvS1kG3d1Psmdh3bxecbPg/1ESo9kimxwEOaoac0Q49ohmZqhv6vM0wZEWHFTZVTqjyhyghne0KVU2fkeZhbArWd93UwEZpgCt1NUcN8oK6INCtHPyoV8bHxjOs5jo/WfsS+E5VjJUatWnDsWLS9sFjC89T8p3jku0dC2izcuZCHv3uYFxa+ENLu+Onj7D+5n4/XfRzUJjc/F0GYt30ea/atCWoDcG6Lc5m+cjrHcwMvWcrNz6VH4x70b9mfSVmTAg5hum2N6DKCBjUaVPmhSc3QfKC1MyxZKiJJnLxJhI3+W2lvGAF3Av8SkW3Ao8A9zvEWwDYfu4CF7ozPcoszpLkwL69adM8BSE9L53TBad5Y9kZ44wogJcWKmyW6rNm3htZPtg4qIC5vrXyL++bcx5Gc4JHk3rmtIAJSzC5Mb2tQu0HESExQoXHbueXsWziWe4x3V78b0C4nL4eE2AQ8qR6W71nO4l2Lg7ZVM6EmN/S8gfdWvxe2B1oF2Ah8J5lyr2TK/7pbpBdHMizZF+jnbBcBTwOvlspVBxH5UkSWB9iGA78B/qCqrYA/YLJClwhVnaiqfVW1b1xc9anq06tJL3o37c2krEnRdgWAmjWhCq+Pt1QD1u5fy9bDW3lp0Ush7XLzczmZd5K3VrwV0sZt84cdwefUXLuZ62YGDePPzc+ldZ3WDOkwhKlLp5JfUDz5vRvheEnbS2hXr11QsczNzyUhNoHregSvFOL6lBCbgCfNQ25+Lm+uCD7fV0XYAHyE0alaPltERDIsud9n26HKk8DQ0npr2tRBqtojwPY+kA649XreAs5x3kdU6K6640nzsCh7Ect2LwtvXM7YnpulPDmdf5p/fvtPjuYcDWrjPtRfXfpqyCCKnHwjJGUxB5aTl0Pbum3JK8jj9WWvB23L7W1tP7I94CJs935JcUlM6DWBWZtmsfXw1sBtxSRQN6kuV3e5OmClEPfzJcQm0Ltpb3o27lmp5udLgzPPVmyL9PpIhiX7+Gx9RbiV8i1yuhO42Hl/CWYBH5hCdxOcqMn+wGFV/dlVFLuh5w1mzVslGFOvWRPy8mzZG0vJ+e/i//LcgudC2izcuZD/99X/CzkH5j7Udx/fzWfrPwtq54rBN1u/Yf2BwDEJbk+qd9PeTFs+jZOnTwZtK61pGmc3OzvoKIo7lHhl5yupm1Q3oJ3rU2JsIhNSJ6AoU7OmBrRLjEsEzI/bQJVCfHtuIsKtfW+lY4OOlbJcVjgkU16STOkZ5FyKZMpNkilhk/5FMiz5mM/2D6APMLokzpaQXwOPiUgW8BAmMhLgY8wY7HpMppTbytGHSkvD5IYM7TSUV5e+GvU1bylO5jk7NGlx2XRwE6kvpLLxYOhp+VeWvMIfv/gjx3KDd/1d4Qo1B+Y+vONj4kMO1+fm5zKgzQAEYUrWlJBt3XL2LRzOOcz7a94PaucO/y3ZtYSsXVlBbZLikri+x/W8u+pdDp86HPB+CbEJtK3XlgFtBgRc8+a2BXBZu8toVrNZsc/qFUpHBG/rdxtTR0z1XlfFeBa4VzJllWTKW5Ipz0mm/Fcy5Rvge8zQ5NvhGolE3H7lZitR5TJVbgHK7eeAqn6rqmeraqqqnquqPznHVVVvdwrd9VTVheXlQ2XHk+oxv1Q3BP+lWhG44maHJi0uK/euZOnupbz0U/g5sOOnjzNj1YyQNm6bP2X/FNLm6i5X88GaDzhwMvDalNz8XNrVbcdl7S9jStYUCrQgaFuD2w+mVe1WIQNBEmITuL7H9QFHUVS1WG/rZN5J3lr5VrF2AK8Apaems+7AOuZtnxfwfgCxMbGM7zWemWtnsvvY7qBtVWU0Q5doho7GxHk8C3yDGbm7WTM0VTP0Kc3QsGXXIhG3QAoZVjUt5ccVHa+gUXKjqA9N1qxpXm3P7edBfkE+Ly58MehwHRQ+ZKcsnRIwiMLfLtKsG6GCLcD0tnLzc5m2fFpQu4TYBNJT09lyeAtfb/46aFtJcUmkp6bz+YbP2XGk+LS+21aD5AZc2fnKYqMo+ZqPol6h6de8H10adin2/9XtmcbFmFmea7pdQ0p8SpHPqqrk5OcUEa30tHTyNb/IfF91EjcXzdBjmqFzNEPf0Ax9TzM0dEisH6GqAnQRYRRQR4SRPpsHSDpDvy1nQHxsPDf0vIEP1nzA/hPRSxFie27Vh+krpgcdrnNZlL2IW2feGjLzvPuQ3Xl0J19t+iqs3ezNs9lyaEtIm26NuvH6stcDJg52bc5pcQ6pTVKDiqA7B3Z1l6upnVg75BxYQmwCE1InUKAFvLq0eGC4b0/Kk2oyB326/tOA7QCICJ5UD99u/bbIfJ/bjogAJoz/mm7X8OaKN70/INwgGV/R6taoG/2a9ysilu66ulg583I31YVQPbfOwDCgLnClz9YHMy9miSJuuG+wX6oVge25VR+eW/Acv/3ktyF7ZSfzzLlIeluxEhvW7vxW5wMEFVVXzG7pcwsHTx3ko7UfBb2f2ytbsHMBK/euDGiXEJtAcnwy13W/jndWvlNsvs+3rY4NOnJBqwvCzoEN6TCERsmNiohloF7UuF7jzJq3JUUFKTE2sUjb6anpHMk5wnur3yvSlr+dJ61opRB/obSErgrwvio3AsNUudFn+50q31egj5YApDVNI7VJalSHJm1ASdXg4MmDZB8NHVick59T5KEa0MYRm1Dll9yhtqGdhvLu6neDJg52U0oNbDMwaOJg98F+RccraF6redje1theY4mLiQtYPcNXkNJT0zl++jhvr3w7aFtgBGTVvlUs2Lmg2Gd0beJj4xnXaxwfrvnQO4oSSNxa1G7BZe0uY8rSwvk+X59cLm5zMa3rtPZ+1mDDjWN6jCEhNsH7A8LtmVZHJFOSS3NdqGHJPzpvbxDhaf+tVF5ayhT3l2q0Slz4Dkvu2RMVF37WqCpvLn8zbLj3nZ/dyYWvXBgwiMLFNztHOBsgaPkl7xxYn1s4lXeK6SumB7TLycshIcZEHG44uIHvtn0XtK3k+GTG9xrPJ+s+KRJE4bYTFxNHjMTQOKUxl3e4vFji4PyCfPI13xvgcX6r8+lQv0PQObD42HgAru12LUlxScV6oP6i5EnzmMxBy98o4re/2HjSPGw9vJU5m+cEbAcgRmKYkDqBLzZ8wfYj24O2Vb9GfW+lkNz83IBtVXUkU86XTFkJrHb2UyVTQq8f8SHUsOQq53Uh8FOAzRJlvL9Uo9R7c4cln3kGmjSBxX5ZgQqCP0stZcCSXUsY884YXlwYuvry/hP72Xhwo/ehGgj3Ifrlxi8DBlH42nSs3zFo+SXX5vxW59O9UfeQEYeJcYmM7DqyWBCFf1vukGO+5vPasteK2fgLTfaxbL7c+KX32OmC0952oHAObM7mOWw6uKlIW/Ex8cSIeSzWSarDyK4jmbZ8GqfyTDrd/IJ8CrSgyDBhrya9SGuaVqQX5Xs/l+Gdh1MnsY7XLpggpaemoyivLn21yOJsfzypHvad2Mcn6z4pEp1ZjXgCGAzsB9AMzQJ+EenFoYYlP3ReJwfaztBpSxnQOKUxV3S8Imx2hvLC7bl98415/dCppXj0KPTvD23awK7AFTosERAqvyGYBL4Qvn5XpJGJ/Zr3o0ALmLq0+EJi33Z+3efXQcsv+a63Sk9N5/tt37N2/9qAdgmxCdRMqMm13a9l+orpnDh9ImBbCbEJdG3UlXNbnMsrS14p8r34C8SwTsNoUKNBkc8aqPczPnV8sTVvgcTGk+oxhULXfBi0Ldfup+yfWL5nedB5shrxNRjTYwzvrHqHozlHi0VBurSv356LzrqISUsmBRVKgMEdBtMkpQmTsiaRW1D9em4AmqHb/A4FD8H1I9Sw5IcifBBsK7W3ljIlPTWd7GPZfLHhiwq/t9tzc/nKCY57/XX44QfYtg3uv7/C3aoWHDp1iKaPNY0o64b7UA1n5z5Ug9l0bdTV+1ANNQd2ZecrzUM1TG8rUBCFr53vHNjR3KO8u6po4mD/Xkt6anqxxMH+guSuP/NNHBxIIM6qcxaXtL2EyVmTQ86BXdL2ElrUahF2DuyGnjd45/tCheWnp6Zz4vQJ3l75dsihxPTUdNbsX8M3W80vx0C9sriYOMb1MpVCdhzZUR3FbZtkyvmASqbES6bcTeGIYlhCDUs+StHsJP6bpRLg/aUahWTKSUngG5y1ebN5fest6NYNrr0WZsyA/Ih/a1lcso9ms+f4Hv7947+D2viGxgebAwPzQG5Qo4H3oRqsLXcObM3+Nfy448eA7QCkxKcwtufYgOWXcvJyvCHpzWo1Y0iHIcXWvKkqpwtOex/Gv2j9C9rUbRM064Zr5wZR+EccBprbysnP8c73hZoD23Rok7dQaKC2YmNimZA6gc/Wf0b20eygbTVKacTQjkOZunSqtwcaSGz6t+xPpwadmJw1OeRQ4rXdr6VGXA1vQuhQIphXkMesTbOqo7jdCtyOqf6yA0hz9iMi1LDk1+4GzAMOAgeAec4xSyXA/aX6/ur3K7zEhQjExha+37nTCNnSpXDeeXDVVbB7NyzzyfGsCh99BBvLs2hSJUdVefjbh9l8aHNQG/chGkn15dZ1WoesvpyTn8O5Lc+lY/2OQX8EuQ/2a7pdQ424GuHnwIKUX/IPSU9PTWf7ke3M3jzba+M/BxYjMaSnpvPVxq/YdnhbkbZiJZbYGPNHVq9GPa7ucrU3iAIgt6B4OH2fZn3o0bhHsd6Wv92ILiOomVAzojkwd74vVK/Mk2YyB3249sOgNu5839dbvmbNvjVBBal2Ym1GdRvF/O3zg7YF0LNJT/o061Nk0Xh1QTN0n2boWM3QJpqhjTVDx2mGRrywN5LEyUMxpQeeBp4B1otweeldtpQ17i/VaJS4cEevunY1SZRXrIC9e6F7d+jXz5xb4hM1/te/wpVXmvPr1hVv7+fAvhP7uOere3jg6weC2rgP0UiqL9/c52Z2H98dtPqyu57Kk+Zh7pa5AfM+ur0I96E6bUVhEIWL7zBhrya96NOsT8Delm9v5KrOV5nEwWHmwLyJg33m+wKJTXpqepHEwYFsRIT01HTmb5/Pmn1rggpSSkIKo7uN5q2Vb3E893hQcevcsLMpFLpkUsgAjys6XkHD5Ia8suSVoDZQON+37sC6kIKUnprufR/KzpPqCWtTlZBM+bdkytPBtkjbiTRx8kBVBqhyMTAQE8ViqSS4v1SjETXpilvnzub1C2fqr1s36NABkpMLxe3gQXjySejd2/T4MjIq3N0KYc/xPSFTT7kPyHDVl8E8sCKpvtwwuWHIFFWJcYmM71U8iMLXxndu69CpQ3yw5oNiNlA4/5Oeml6s/JK/QCTFJTGm+xhmrJrhLRQaaA6sXb12/KL1L4qseQskNr9s/0ua1mzq/VsPJkhje44lVmK9w3/+93PxpHk4lnuMGatmBA3wACMgK/auYN62eUHbSohN4IYeN3h72sHaalm7JYPaDQppAzCwzUBa1W4V1u76nibHZXURN4JH6JcoUj8ScTuqim+NiI1A8AJLlgrH95fq6n2ro+JDly7m9VszfUG7dkbAevYsHJb88ks4ccIsHZgwAd5/3+xXJwq0gM7PdOaBueF7Ze5DNZTNqK6jWL5nOYuyFwW1qZlQkxt63MD7a94PmDjYFYBWdVpxabtLiwRRQGGiX/fh6D5UA63vgsIHbaDyS8HmwHwLhQbNupHqYe3+td6huEALk+Ni4kzi4HUz2XN8T9DFy81qNWNwh8FMyZoScg7swrMupF29dmHnwNxCoRMXTTS+B7HzpHm870P2thy7UDbufF84u4bJDbnrvLsY0n5IUJuqhGbo5FBbpO1EIm4LRfhYBI8I6cCHwAI312SpP4GlTPH+Ug0RWFAeuB2K9u3N6xontWnDhua1TRsTNQlmyUByshmuHDnSCNvs2UWaY9MmeOop2BI43WClJycvh0OnDvHyopeD9t6KLIYO0tt2e3fje40nMTYxoJ2v2HirLy8vPjSdm2+KXYIRkM2HNvPNlm+85/0T/XqDKDZ8xs6jO4u045u/sGFyQ4Z1GlYkcXCg3s85Lc6hS8MuYSMOr+l2DcnxyRHNgbmFQkNFHHpSPew4usO7ZCHYHFh6ajqzNs1i/YH1Qduqm1SXEV1HMHfL3KBtgckc1KtJLyC4AIKpYlAroRY14moEtQG445w7+J+z/4fUJqkh7f4x6B/cc9E9IW2qGpIpH0qmfOC3TZVM+b1kStj8xpGIWxKwG1NAdACwF6iByTM57Ax8t5QhbmRasJL25YUrbo0bm9cdOyAmBurUMfstW8L27cbuhx/gnHMgPt4EnMTEmGMuhw6Z43feaV4PVmx8TETc9dldxQpF+uI+uHcc3RE0cbBrk9oklVmbZgVMHOzaNE5p7A2i8E8c7Dv/4z5Ug6Woch/GI7qOoFZCrSJ2gYYJ3cTBvvN9gfIXpqemFym/FGoOzE0cHEzcaiXWYlTXUd7EwcHWbnVv3J2+zft6e1vBhObKzldSL6ke/1n8HyC42LjzfSv3rjzjOTAR4ca0GwETFBKM5Phk3hn9Dn++6M9BbQCa1mzKC8NeoEZ8aBGspmwEjmHqd74EHMGMGnZy9kMSVtz88kr6bzedofOWMiQ9NT3kQ7U8ccXtyBGoV88IFxhxO3kSDhyADRsK5+ZSUkxQyUKfqnzPPWeiK//9b7P4+8knK/YzRMILP73AX2f/Nej5iHpljpjc3OfmYkEU/u24vbIDJw8wc13o6svpqen8uONHVu1dVczOfRgnxyczuvto3lrxljdxcCCx6dSgE+e3Or/IfF8gIfEvvxRMbMb3Gk+MxDAla0rYOTC3UGioYUJPqikUumzPsqA2SXFJjOkxhu1Htge9H0Cbum0Y0GZASBswhUKb12oe1u6Oc+7g2xu/5aw6ZwW1Abis/WX0adYnpM3PnPM1Q2/QDP3Q2cYB/TRDb8ck8A9JJNGSbUV4XIQZZbWIW0SuFZEVIlIgIn39zt0jIutFZI2IDPY5PsQ5tl5E/t+Z3L+64v5SDZWJorxwhyEB6tcvfN+ypXldtQr27zfDlC5nn100kvKdd0xmkzvugEGDYPLkwp5hRbB8z/KgYfcuufm5LNm1JGjiYPfBXTuxNjNWzShWfdnXpnODziGrL4PpbbgPVX+x9A/w8A2i8MV/XsqT5ilSKDTUHNjKvStZuHOh187/oR4fG8/YnmO95ZfcyEx/3MTBk7Mme6MwAwnEgDYDOKvOWUxaMilkr2xMjzHEx8Rz6NShiOa2gt3P97OGs3ELhYazi4uJ44KzLgh6vroT7PkuIm1E5KSILHG2F8I0VVMyxfsLwXnvpo4IWzA7kmHJ94DNwL8pu0Xcy4GRwFzfgyLSDRgDdAeGAM+JSKyIxGIqsl4OdAOud2wtPnhL2q8uXtK+vKlVy8ynQWBxc1N0tW1beK5DB8jONnNv+/bBokVmmQDAddeZebfVfvExkydDo0YwfHjZB6OMmj4Kz3ueoOcLtMC7lixc4uBxPccFTRzsnzNx/YH1fL/t+6A2sTGxjOs5Lmj15fgYk+i3Sc0mXNHxiiJD0/7BIgAXtLqA9vXaF5nbcu/ly+juo0mKS/KKZbDgjfS0dHLzTfmlcFk3th7e6l2yEMguRmKY0GsCX2z8gk0HNwVtq0FyA67qfFXQdlz6Ne9H14Zdw9qN6jaKlPiUgMLsyx3n3MF13a+jR+MeIe1+5gR8vjtsUNU0Z7s1TDt3Ad9KpsyWTJkDfAPcLZmSAuFTQEYibqdUeVqV2X4Lu0uNqq5SDVhVdTgwTVVzVHUTsB44x9nWq+pGVc0Fpjm2Fj/S09JDZmMvL2rWLBS3Bg0Kj7s9ukVOsJ+vuLnvN2+GrCzz3l0bN2CAef3a5y9tyxb49a/NfT74AB58sCw/ARw+dZgvNn4RNnEwEDZx8AVnXUDXhl3DBoIEqr7sbwPBqy/Hx8QXmwPbeXSnN3GwGyziO3TnDmHO3jybzYc2BxW3Okl1GNFlhLdQaLA5MN/yS6HEzS0U+vLilwPez/sZ0tIp0AKydmedccShiHjtaibUDGpXM6EmU0ZM4e7z7w5qAyaMf9o100LOp/3cCfF8L1k7Gfox0BG4E/g90FkzdKZm6HHN0LCTFnER3OMpETKAzwHvjLYqxWOTz5wWwHyf/e3OMYBtfsfPDdaIiNwC3ALQokUL5syZU7ZeVmJUldbJrXlq7lN0PNqx3O83daqZJ/vxR/jb3yA31/Tc3K88Lw8efRRq1DBDjocPF56rU8ecW7cOTp0y70UKzz/1lEnx5e7v3An/+IdZXrB9u5nfmz27aAqwYDyx9glqxdfi5rY3B7U5kXOCAi3g/vfu5/qzri92/nieWZOWVieNJYeX8Mi7j3BBw6LDTxuPmQXS61av46JaFzFx00Re/fhVWia39Nr8tM8s1Vm6ZCk5tXK4sP6FvLH0DUaljCIp1gSBLd9hckUumL+AOvEmOqdLrS48+92z9M7pDcD6zeuJJbbI33etglrUjqvNI58/QuL2RE7lmyHAbZu3Maeg0K7jKfO3cf/79/OLhibR+ro165hzsNAGoLf05o1Tb/Dwuw+zfe92CnILAv5/urDmhTy74VnqxNehY82OQf/PXVT/ImZmm7nDlctWErs1cOXoHrV7sPzIck4cORG0raSCJBolNiLnQE7I/+NpBWk81OMhtmZtZStbg9rVpz555DFnY/C2fkbEiYjPjDgTVXViGbTbVkQWY4JD/qqq34SxPxtog9GqVMkUNENDl4x3UdWQG+g/QLeDfg0629lmhb+OLzHdU/9tuI/NHKCvz/4zwDif/f8A1zjbyz7HxwPPhPNBVUlOTtafG//89p/KfejafWvL/V4FBWZTVe3cWRVUf/e7wvPHjpljjRqZ1yNHCs9lZ5tjTz+teuONqk2aFG37sstU+/Yt3O/ZU/XSS837zz83177zTtFrDh5UHTxYtV071ZkzC493f7a7pvw9RY/mHA36WVL+nqLch3Z5posWuB/Kh73H9yr3oY9//7g2/ldjHfnmyGI2C3YsUO5DP1j9gW4/vF1jMmP0L1/9pYjN2yveVu5Dl+5aqqqqszbOUu5DX1v6mtfm0e8eVe5DD5867D327I/PKvehi7MXq6rqHTPv0HoP1yvmwx0z79DEBxL14MmDevDkQeU+9Il5TxSzGzhpoLZ7qp0u2rlIuQ+dsXJGMZu8/Dxt8VgLHfb6MB355kjt8VyPgN/d7mO7Ne7+OOU+dOhrQwPaqKp+t/U75T6U+9Aftv8Q1O6ln15S7kMHTx0c1Ma977GcYyFtLCUHOK5l/3xPBBo478/GdFhqB73HfUzlPr7nPp7jPv7tbE+H8st3i2RY8lqgnSoXqzLQ2S6JQDQHqWqPANv7IS7bAbTy2W/pHAt23BIAbzb2CshYIlLYcwo0LJmcbM7v3Wv23TI5YGrAxcaayMjt26F166Jtd+1q5txUTbTlsmWFw5UDBpi2Zs0qes3dd5ssKSdPwpgxplcJZggvUPVlX3Lzc2lWsxmr960OnTg4IYVxPU31Zf/Ewb5DfC1qt+CX7X/JlKwpRRZN+w8DeqsvB0hR5TsH5F99OVg0YXpaujdxcLjIxI0HNzJrk/kSA7XlBlF8su4Tth7eGnQI0C2/FKwdl/NankfH+h2D+uTiFgoNl3WjcUpjUhJSQtpYyp7SPN/VTDftd97/hEnr2CnEbfoCF2iG3qYZ+ltn+12kPkYibsuBupE2eIZ8AIwRkUQRaYsZb/0RWAB0FJG2IpKACTqxZXeC0LxW84AP1fImUECJSGFpnFq1CpcIuOfq1TPClZ0NzZoVba9LF1Ple8eOwvVwF15oXuPj4fzzCwNVwLQzdSr85jdG9I4eheefN+fch7yJTDRDnUUTOptM9dd1v44acTXCzpWFShwMRVNUbTuyjdmbCler++cndBMHf7nxS2/YujdYxKkKDQGqLweZAzu72dl0b9TdG3Hoey9fRnUdRc2EmuEzzzvzfQt3LjzjiEN3vg8IuYC5TlIdXhz2Irf3izgJvKWSIyKNnOBARKQd5vkeKoX6cqBpae8XibjVBVaL8FkZLgUYISLbgfOAmSLyGYCqrgCmAyuBT4HbVTVfVfOAO4DPMPV8pju2liB4Uj3FHqrlTSBxAyNqvq++1KtnFmsHEreOzpThhg2FmU96+ASpnX++Eajdh47Q6/lePDZ9Hrm5cNNNRhgHDYJp04xtTn4OsRLLnM1zuOVPmxg4ENLS4E0noYcrAo1SGjGy60jeWP5G8cTBeUUTB/du2jtoeL77gPdWX/ZZNO0vgOCTODhrqtcmLibOWxXaxb/6cqisG/O2z/PmfQxkl5KQwjXdrmHN/jVBbQC6NOzCuS3ODWkDMLTTUBqnNKZeUr2gNgB/OO8PvDriVTo1CPWj3XwngzsMDmljqXwEe75jqmgvFZElwNvArapaPF9cIQ2BlZIpn/lmKYnUj0jELQMYATxEGS0FUNV3VbWlqiaqahNVHexz7u+q2l5VO6vqJz7HP1bVTs65v5/J/X8ODO9S/KFa3gQaloRCUasdIMCsfn0jbPv3Fxc3d3/XLli/3gSg+LbdrZsZsvx22TaW7VnG6xufpEEDk5gZYNgwI4qbNhmxGNppKILwnwVTGD0a+vQxa+qOHy8enn/o1CFeW/Ahp30CIv2Fy5Nmqi/7Jw72tfFWX175jjdxcKDelLf6srNoOlgS3yLVl0NEJrpD0+F6ZW5vK5SN+1nD2STEJvDjzT/y90tC//dMjk9mbK+xRaI8LdWHYM93VX1HVburWQbQR1U/DNPUfcDVlFJ7IslQ8rUWXQKQD4yO9AaW6OBmZ/B9qJY3NZxRJv+em++wpD/165sF3lBc3Jo6AxKuuHXoUDQysqtZvsTayU2N9QAAIABJREFUDUYstiS9T9+LDnptLrvMvM6dawSlQ70OnJV/KZo6maeeLuDRR836unffLTrHdUnbS6hV0JKbn5lEt26w1Qmw8xel63tcb6ov+yUO9rUBMzR5Mu+kd74vVPHMtfvX8sOOH4IKV6TVl910bG5m/2Drty5qfRFt67YN6I8v13U3iYPDzYG1rtuaejVC99wslkjQDP3ad6OE2hNJzw0ReovwLxE2Aw9QglLflujhZmMPFURRlpSm51avXmGwib+41asHcXEmKGTDhsLkzC4dO5o5vPWbjFhobA5xaYWJgzt3Nj4tXlyYXeP0gnSot4m1p77lF78wGVOmTy8qON99G8vRbydAx0/ZeTSb3//etOcf5FEvsREXNBzG1KxXvYu7AwmXW3053KJp30KhwTJ9QGHi4Hnb54XNhZiv+QHv5RIjMd7M80lxwXPR1qtRj8cHP86vev8qqI3FUtZIpvSWTPmXZMpmSqg9QcVNhE4iZIiwGpOdZCsgTrTkM2fqtKX8ObfFuUUequVNsDm3cD03F//zMTEmonLB4Y/YdPHFNGleNONOUhI0bw7Ze5zllwWxrE6a5D0fGwu9esHiJSZLx+lTCez8agSJ1GLSkkmImN7d3LlwMrdQcJ59FuptmwBSwAW/eY333jOVw31FKT8fhgyBr5/ysOfEbiZ/W5g42LVxcasvf7P1GzYc2FAss4iLt1Do8mkczjkcVJDc6ssQvEcGhYVC/f3x5w/9/8Azlz/jzeQRjNv63cbIrrYQiKV8kUzpJJmSIZlSVHsydKBmaMTaE6rnthq4BBimyoWq/BvTLbRUEfwfquVN7dqQkFC8hxZuzs0lMcBzumlT2HDqR/JbzWVv3eLZ+Js2hb0HHNFbO4wNp34oUtOuVy9YvtJMnO3bkwCnU7ikSWH15QEDzMLyZSud3l9+Ah9+CGMGdea8luexpe4kQJk2rahw/ec/8NVXcMeQy+FEQ/42YxJQPMP+ggXg8UDS2sJCobn5ucRKLLExxRcwe1JN4uCP1318xpGJbjq2cHZ1kupw+zm32zkwS2WhUHsy9ELN0FJpTyhxGwlkA7NFeEmESwH711/FcEvaB6q+XNbcfjt89FHRcH8o7LnFBkhGUc9neiaQuDVpAvsOGlFZEV88PL9pUzhw2BG3xTcVq2nXpk3h+b3Z5gF/2wXp3kKhZ59t7JavNqK0fUsCJ0/C5Zc7c2CHVtB5wCK+/LJoCP+LL5qAlKefSODcGmPZWesDlq47UGTocutWGDjQ5MP835tb0r3GIG/iYF+xyc01hVy3b4eBbU2h0CM5R0IKUqTVl2/rdxudG3SmQ/0OIe0slkpEofZkykuSKaXSnqDipsp7qowBugCzMfm9GovwvAi/LKXTlgrGLWnvX325PGjevDCIw5fhThbQdu2Kn3OHMsEMM/pTqxYcPWEEY+VpU33ZF19xq5F7FkM6DGHK0inexMGtWwOxjrjtSqB5cxjaw1RfnpQ1ifbtTW/TDUrZuNaIRf/+JnFwYmwiif0nMX8+nMgxNruzE1i0CMaPNwEu917pgbhcMt6aVqR39+CDkJ9v5gvPOQd2fuxhy+EtfLHxC+8ygLw8GDrUfG+dO8Oin2KCVl/OzjaRnWAKhT4w8AHG9BhT/EvzoUfjHqy+YzUtarcIaWexVBY0Q9/TDC2uPZnyvGRKxNoTSbTkcVVeV+VKTGaQxcCfSum3JQp40sxD9evNZ5TvutQMHWqSI98dICdtjRrAFXfARX8P2HNLSsKIU0EM+eQVSRwMRtwOHzOCUq+2qX228+hOb027s87CK26HDyTSurVP4uBNs9lxbAudO8OGzY64rUukbVtTecCtvrwx5XVO5uZ4A1eWLTaO/tL5bzb07DQSD/Vi9sHCRdOal8D06TB6tBH1P/4RDnx/NTViarF091KvcL36qum1/fnPZoj217+GCb3MImdfcXvwQfPjoUULM9QJ8KcL/+QVQpc9ewoDdCyWqoyTIPl1zdBSaU9E0ZLemykHVZmoyqUl9NMSRdyS9hWRjisYrVsH7pklJQHtvoALHqEg9mSx8zVqYMTpeBO61u5b7DM0bQoaYwSlbq0EruxUtKadb8/t8IEEbwke76LppVPp0gW2bDc22dsT6NKlsH1Pqodj+Qeg00y27jA2WYsSqFuXInZ9YjwcrrmAxdmmztsP8xI4fNiIG5hSPjUTk2lz/DqgULiefRZSU414ZWSY+na7V3XkkraX0LSmWQuxaBHce68ZKq1Tx/QY8wPMQLz+uikx1Ly5eR+MdeuMoOblBbexWCoTmqEHNUMnaoZGrD0lEjdL1SQ5Ppnrul/H2yvf9lZfrix4xSvpCLN2vlfsvOm55UB+Atd29BQrFPr/2zvz+CiqrO//TnYIJOxhJ7IJRCAgKoqoqOAyiKigKEqizouOMvq8Ko7K+IyzqI/jjDOPOjPvMDoDCrjgKOBHEGXAFZFFEpaEJYQl7AYIOwlJzvvHqdtV3V1VqZBOdyd9v59PPp2uul11b3VSvzrnnntOmzbGfoi4JSck+9W069ABPnE7UpqEToZ3LrNFJkZkjsDM/Jno2Il983r7dif5uU+v7X4tOjbrCGTPwJ790qZwQxKGDPGfW7zrgolAVQI+3PQh4igO338XDyIzXVhSkvx+anmuvI9Pwq5dUon8rrvEvXnXXTLejz4C5k+Yj3duk9Ref/6zuGffeQf44x9lYfrHActfDxwAHnxQFrBfeCHwwAPixgxk4UJZHzhyJDB+PFDt4KlesgR49FEz7ZlG09DQ4hYjqOrL/y74d6S74odP3ADM3RJsWfrckpXJmJgt1ZetASPWz7dKNzOHqJp2CQlAarqIX8Vp03ID4CsUWtFuOc6clTYnjyX51ZyLj4vHPQPvAXotRHGpVF3aXpSE3gGZo4b2bwcU3YCzRr7H774DsrLE0lJceSWw8+vL0D29J5Lik3xJn3/yE3lt2lTaLFok9cVSk1Jx9qzUrhs/Xo41dqysI5w71//8M2ZILs2ZM4G33pKcnP/8p3+bigoRwH79gKlTgXnzgA8/DP5OvvhCXK6vvgpccYV/Dk4rf/2rPFxcc43k9bSDGZg/X6qsOwmpRlMfaHGLES7rchl6tuoZ1nRcXrDOqS3dGVwo1CdeVUno3r61L3GwKhTqJ25pMhc2pOMQ9GvbzzfWZulGNGWVabkBZvXlguQZvmOgKiko8CVnYA4QV4WiZiKqx8uS0DMg+LBPHwB5uQDEKtuwwUwDphg4EAAIj3Sfjt+O+C1WrRKLrK9ledmIEVIJ4cgReb9ihSxVuFES7iMhQVycCxf6i8WcOcCwYdKP3r2lasKsWf7nnzcPKCkBXnpJ6uL16gX86U8I4vHHpZBscbEs35g6NbjNd99JdGzXrsA334ho2jF1qgjyuHGS0NqO48eBMWNEtP/nf+zbAOJOfeYZsWzd2LlTgng0sY0WtxhBBVF8seMLbD+yPdLd8eETp603opqr8fa6t/32+8SvKgkJCWKV/XjqRywqWuT/eQCtWojlpsa6vGQ5th7aitQ0Q7gqk32VwQGxjsb1G4dVp94DUsp8bbpYiysB6Nu2L9qUX4yKpjt9bQLFrVkzoNPJ0Uiuao3k+GTs3o2gNllZ8pq8d4Scd5W4EK3uTSWIeYbndYVRuleV+gGA4cOBsjIzmXRpKbBunSmAgOTV3LRJKioo5s2TQJlRo2RZxqRJwPLl/u7LtWtlju+xx0TgHn4Y+OwzEQwrL74oVtvXXwNPPSWWZEGBf5sNG4BXXpFE1j//OTB9ur2b84kngE8+kev19NPAp58Gt9m3T9y6L74I3HqrWKd2/Otf0m8n4QakMO7EiUC7djLPKeXFglm7VsomPfss/HKMWmEGZs8GXn7Z2XoFJNBnzhx5uHBj7drg6xjI8eNSud5u3lVRUWGWe4pVtLjFEJMGTgKBggQkkvjE68d+uLzr5UZJGvNu4xOvahGu63pI4mAVWOL7PIA2LczoQmtNu6bNTassNaD0V252Lk5VHQey3ve1CcywAgCDKdd8Y2PdAUD3bklov2sKujfrDyA4XViXLmKpbdwoN8XCQllkbkWsO1Pc8vIkSMSa0mzoUHlVQvHNN/J65ZVmmxEj5HWZURSCGVi8WFygar3hmDHyahWTBQtk/m+CscLgnnvks9Y5vrIycZ3ee6/U1Hv4YRHo98zMZwCAf/xD5hpffhl4/nkZ++sB+SX27hX36UMPiVD26AH85jcI4oUXRDzWrpVqEE8+KTX7rOzZI4mwhw8HrrtOrMYtW4KP9atfidBkZsq5PrDJTrdvnzwEzJsnwT52kb6AfP7uu6U/o0YB5eXBbfbulTWREyfKA86mTcFtAKlkP3iwtPnrX+3bbN0qwp2dLRlyKiqC2zDLA8XQoeKejlW0uMUQXdO7YsR5IzAjb0ZY67wBwLNLn8U3u74J2p6SwkCCBIzkDMwJKhSqxIsMcUuMT8TE/hN9hUKbNIF8HkDrlgm+z1lr2jVpbpSuqUryW1cHAFd0uwJdm2cCPRf72rS0yft7ResJQGUSCHEAx6Ndu+A27dsDKSuew7TOsgwh0HIjkpv3zp1yoz5xAn7ze4AsWm/b1nx6z883BU/Rp4/Mz+Xny/u1a0Vc1IJ0QD6TkiL7AKmMcPgwcOmlZpsLLhC340pLXdZly8R6VGLao4eIgLUo7OLFEmk5dqy8b9dOBGW+pUwls1hzo0fLEofmzWXe8OOP/aM033tP3j/6qAjh5Mni8ty61WxTXi5ziXfdJTf13/xGrJL5AWUx33hDBO+f/5Q5yPh4mTe0UlYGvPaaiPby5fJw8etfB1tvf/yjuIbXrhV36l/+Im5aKyUlIroTJsg41qyR8wbyzDNiXc+eLWOcMiW4TV6eiOjtt0uwz+OPmwm7rUyeLFbk1KkS9BP4sACIWM+eDfz0p2YChVhEi1uMkTswF9vLttsKTX3ywjcvYNrSaUHbE5OrAGKgKhnj+433JQ5WKMtNiRsAv0KhZsBJElq29E9ioGraHWtnmCY2llscxeHu/pOkDwBQneQXBKI4r31LYPPNiK+SA9gJYEaG3HSVCy8zM7hNhw7yJL/d8AwHihsgyxdKSmRObcsW053p63OciI6aV9q0SY5jXWoRHy9zeRuNqodqbdyQIf7HGTLE3FdVJdag1QIExCX69demACxfLhbbJZeYbUaMENfo8ePyfutWsX6uv95sc/31Mn9oFdNPP5V+qgcBtXTCak0uWSLHVdbkVVfJtQ4MhpkzR/b16CH7x46VG711bvL990UAH31U5i8ffliu0Zo1ZpvKSnFv3nab9O2ZZ+QYs2f7n2/mTLGcXnxRhPuii0QErRw6JFGukyeLOE+dKmnb1PeieO01eWCZPl0s3vJyebWyZo0E+/zyl8Dvfw9cfbW4XgMDdV5+WR6AnnoKMY0Wtxjj1r63ollSM7+Iw/qmqroK1VyNr3Z+heIj/o+/8UmmyzA9JR239r0V725811coNCUFQEK5n7gNyBiAwR0GY2b+TEvASXJQ4mVV025ni5m+cwSKGwDcO9hcCJ3WNCkofRggN0ss+l90Wf4R0tPlxmjXpqxMxAsIro4AyBq0vXtlUTtgL4Bdu8pT+8GD8pTetWtwmx49pAwQIOJmDUpRZGWZN9H160XwrMVeAbH21q0TYSsulvkoO1dpaan0B5DlC4MG+adTGzpUxE8J5Vdfyevw4WYb5SpdvlxeKyulnVoMr65Ht27m5wGxJpOT5WYOyHlvvFFEQgluSYk8CChXKyAu2AMHTBcvIFZnly7i/gNEwIhku+Lbb8XKVULbubPM9/07INB47lzZnpkpx7jzTrnO1mCWefNEAO+9V97n5prbFZWVcmwVEdutm1yTwLWKH3wgf3fqWD/9qaRs+8bynLp9u3wH999vn+4uloiIuBHReCLaSETVRDTEsn0kEa0hovXG69WWfRca24uI6FXSWV7PidSkVIzvNx7vF7yPkxUnw3JOlbUDQFCOy7hEU9wA+AqFqjpkyjKLY/9UVDkDc7Bm3xpsP7neF3ASKG6qpt2ZuEPGOZKD3JIA0LN1D8SVyF24RZp9rsaMDAAnOmD3V9fYzskBZv25TZvgKIAdO5olfAB3cVPBB9blC74+9xQxqqyUm/r55we36ddPbn4nT8pNr0sXcYtZ6dVLBHT3btMV2q+ffxsliBs3igiuXevvAgXEagEkGAUQQUlLg9+SiTZtZCxKbDZvFjG1WpOACOK335rvV64UMbVmsBk6VARIuQq//lper7jCbKNSwal91dXiXh050qwL2Lq1iLmamwSkTVycv+iOHCkPAWVG3NGhQ/L+hhvMNjfdJK+ffWZuW7ZM/nays+V9RoZcu0WLzDarVvlHxALye3Gx+RAESNDN8OFACyn0gNGjpZ+ff262WbhQXm+5BTFPpCy3DZDkmF8FbC8FcBMz9weQA8Aa+fA3AP8HQC/j53pozonc7Fxf4uBwYBW3wByXlOAvblefdzU6p3X2uSaVZRZX7Z+b667+dyExLhFzt870iZtd1QFVQVqdw85yA4BmP/wS2DQGrZo3sd2vjn32bHBJH0VGhrwWFDi36dBBbrKFhfYVFAARt5MnzfVlduLWvbu4rjZuFIHo1i24jfqcshTtXKAqMGb7dlPcAq1A5RbdsEGOdfp0sAC2aiXiZbUm+/TxLy4LiBWo5gqVyAXOKQ4YIC7NsjIR0zVr/F2ggOTqBMygmlWr5G/FeqwOHeSBQ51n5045ZuCxLr9cjqOswNWrZXzWh6XLLpP9KnpVvQ4bZrbp0UOugbJeAeDLL8XNa70OV18tbVQE5pdfmtsV1xh5OJToHjsmfw/K+gWkf4MGmeKt+tWhg30e11gjIuLGzIXMvNlm+1pmNpw62AigCRElE1EHAGnMvIIllO4tSPlxzTlweVdJHByudFxK3C7tfCl2lO3A1zvN/8az1f7iFh8Xj0kDJmHxtsXYd3yfo+XWpmkbjO49Gu8WzAIST9paboDUtMuIP993DjtrCgBaHBoFvDsfrVra/0tYrQYn4VJBJps3O7fp2FFeCwrEurPzP6i1eGpuyk7c1LmUINkFuKhzKXGzsxKV4CkroV274Lp67dvLHNuOHe7uVKurdPNm//Rkiv79RfiqquRmnZgY3E6937RJrNdTp4LdqVlZYrWoKu6bNon1GuiKy842g2rWrZPXQLdr//4S3LNrlwjY6tXB1qQSUzU3t3KlOWepIBILVolbaalYxOqzikGDxFWpoibz8+V6Wt3YffsGBw0Bwf0aNkz6oubdVqwQq1b7taJ7zu02AD8wczmATgB2W/btNrbZQkSTiWg1Ea2u1An0goijOEwaMAlLty/FzrKdNX+gjqhSMXdk3YHmSc39FpL7rLoq/4CRaq7GrHWzTMuNg92FOQNzcODkAYl0dBA3IsLINvcDVQlAuU0DA+WutAsUAfzFzW4uzXoMwFnc1PF37jTdS4Go7YWF4tps2za4jeqDF3Hbvl0Ezs5y69JFBKG4WKylwGrogNwovcwV9uwp4nbypNzU7Vyl3bqJK3X/fjlWt24icFaU5VhYaLpvA5dVJCaKhav2Fxbazzv27y/XqLpaxI3IXigBsYIPHZK5xUABbN5croGK4tyyRa5BoCdg0CDpy9mzpvAGWrnKRaksynXrgs8XFyf92rBB3it3b6A7+IILxJLetUseAoqKzPnEWKfexI2IlhDRBpufmz18NgvASwAeOJdzM/N0Zh7CzEMSnB7VYxxr4uD6RglYi5QWuD3rdszdONeX41IV97SKW+/WvXFp50sxI38GkpPZUdxu7HUj2jZtCzQ95ChuAHB/v8eAv60DzjgoFwz3J5zFzRqJ6KWNk7gpATx40FnclKuypETa2AW4KHFTN1DlErWihErdRO2EKzFRhPHAAREcNW8YSKC4OQW57Nrl3kYtkC8pEQG0s0rPO08Et6jIWdzUtm3b5Ka+c6e9pZiZKUJz4IAIeMeOweHxStwKCsy+27n1zj/fXDen1pvZ9amqSsbnJG69e8t1LygQC27zZhHhQKzitnWr/E0FPsSoB4jNm835x8AlKLFKvYkbM1/LzBfY/Mx3+xwRdQbwEYBJzKzijvZASh4oOhvbNOfIeS3Pw1WZVwUtmq4PrDXOVI5LNd/ns9wq/efUcrNzUfBjAQqPrQHiKxCPYHFTa97U5+1K5gBAh4x4oNTmsd5CTZabNRDDSUS9uC6t1l1N4rZ3b7BloPBiuaWni2irNk5ja9lSgjO8iNuuXcZ6PpsKD+3bi1tPnc9OcK3iVlKCoGwwgAibEtziYhGCTjZ+GiVuuw2fjlNwDiD93r3b/nwtW8o1LylxX6LRu7eIG7OzuClRLC4WwWnSJPic8fEi6iUlsvC8qspevPv0kWtw/LizW9nqwlUuYS1uQlS5JYmoBYBPADzFzL54KWbeB+AYEQ01oiQnAXAVSU3NqMTBy0uW1+t5fNWpE5IxrMsw9GjZwxcwovalpviL1+1ZtyMlIQXzts8A4iuC9itysqX2GaqSHOcZ7G6ygdQkbnFx5nxOE/uYE0/WnVXc7NbTAaa4lZc7L8K1Wm7x8fZiSiRjV3M7Tn1q1cq7uB0+7OyWDbQm7QRX3eh37pQbu53YANKP/ftFADp3tg9rz8wUN6ISJLu+B4qbnaUIyPj27XN3u3bvLufbuVMCPOxExBqgs2ePnM/O8lYRsUqY7cRbbVP9sutT27bysLVtm7uVG4tEainALUS0G8ClAD4hIrXKZAqAngD+m4jyjB/1L/IQgDcAFAHYBmBR4HE1tWNcv3FITUz1WzRdVwp+LEDZmTK/bVbLzVcodMcy7Cjb4ds3a6a/eLVIaYGxfcZiYckcUGI5xt1iL27Z7bOBvYOB0w6mEpxFxEpNbkm7toFYxc3JuquN5QY4i1tysmnVtWljfwNVn1fr7twst6Iicd/ZuS4B2X76tPtcoRdrskULuQarV4vF4iQ2akH84cPwywdqRW13c81axU0JpR1qcf2OHaYlF4gaj3IV2l2rTp3E0iwudn9Y6NrVdM0C9v1Sc6Z79jiLG5Gc4+BBEdT0dG9/w7FApKIlP2LmzsyczMwZzHydsf13zJzKzNmWn4PGvtWGW7MHM0/h+valxQAqcfD7Be/j1NlTITnmyLdH4uGFD/tts4obAF/16Lfz3/bta2mzvix3YC6OnDkCRjVapzv4HAFg9kJgwZuOu71EjtVkuQFmRJqTuFndknbr6QK3O4mEVdDc0icpMXHqjzqfSrDrFuSibrJOQqI+60XclNjYBcIQyefV/JBdG0CEav9+ETenfgeKqZ24pafLQ0BBgQS6uInbvn0iEk6CpPqqzmd3reLjZXtpqbu4deki11yl2HITt/Xr5cHCbrkHIKJ78KCc0+6BIlaJKrekJvzkDMzBsfJjmLcpuFDouVB2pgwfFn7oZ72poBElbt1adPMVClWRlGqflWu7X4uOzTs67vdxMgM44XAXMWjd2j+zfiBexE09TjmJiXVezkncrJ91Eom4OFPU3MRN3fQDF2Y7nc/NLamoSZRLS50tYavYtG5tv4gdEKtIWZNOFm779mK5HTrkLG5qe2GhiKad2CgxVa5ZJ8tUuV2PHHH+XpS4KfF2ehBo2VKO4yZunTpJ1GhenlwDO0vRKm5u51NW7qFDzi7jWESLW4xzZeaV6JbeLWSuyfLKcl+hUEWg5QZIwMi2I9uwbPuyoH2K+Lh43DPgHsf9irlzJWegG6Wl/lkoAgmFW9JqITq1sc4dOd1EAfNm5yZuyg3qJm5KlOLjnYXEOmanoBwvFqe6sVZWOltkgPSjJnHLyBA36bZt3iy3tm3dxVSV/rETEUBE78wZd8s00HJzGmOrVjK+o0edxVQ9IBQVOc8JN29uWp2Ac7+U5abFzR8tbjFOHMUhZ2AOlhQvwe5ju2v+gAtV1VWoYvGBWReI+wJK4s075619b0VqYir+lfcvAM7ilZudiziKQ8smzqozbpzk5asLXiw3hZsbMPB4brjdiNRN2ClaEjBFLXCdmF0/WrRwds9ax2wXBWk9jjqWUxv1ebfxN29uWsFO4m0VvZost8OH3cU0Lc2sWed0PiU2avmFHYEL552+v5Yt3ecBVZ8AEUEngVfWqHLhOlnMGRkibAcOaHGzosVNY655y6/bmrez1ZJPqF1qOywvWY4th2RRkJ3l1iypGcZnjcePp34EIJGUdvRp0wfrf7Yed15wZ536VhOhFjcvbZzcTIA3y02JmxfLzW1ctRU3twAddXN165P1Zl6XZRXWG7mbmKalmXXPnK6n+r7Ky53FLTVV2p04Icd0GmOrVmYOSqfr7mW5ByDj2r9ffnfqV0aGPCzs3avFzYoWNw16tOqB4V2HY0b+jDqteVMiNmnAJCkUalQesBM3QAJGFG5ux35t+zmKX6i47DJJjhtOy81N3NQNPRziZhWYulhugHlt6ipuXhbEp6SY/XJypwLeok+9zIVa++L23Xl5WLCO2+07tgqf00OFtb9a3Ey0uGkASGDJlkNb8P2e78/5GErEurXohut6XIe31r2Fquoqx6CR4d2GI7NFpu2+cDNqlGRzdwqptxIqy83tRqTWS7ndtGvjlnQTN6tw1VXcVH/dxM3rUgeFF6uzruLmdXxqXG7Wq1WMna6nlz557Zf1WmtxM9HipgEAjM8KLhRaW6xRkbnZudh9bDeW7VjmaLnFURzuzb4X8RSPZkkNp2RwOObc1EJcFXhhR6gsN6t1UFdx8yK4ympJSHAWJev2mtx21vPaEUrLTQWtuI3Pi+VWW3GLj3e+Dtax6zVuJlrcNACAtOQ03NbvNry7wSwUWlusIjbm/DFokdICM/Jm+GUoCeTpy5/G6smrkZbsEMYWhYTKcnMTQFWWxSmUHPAmbl6iQGtrubmJspc+KXFLSnIOcrH2w+1aqnZeLTcngfAq3kq+mCtLAAAUeElEQVTc3FLWeok+ra24OVWQAPyvtZe/u1hBi5vGR87AHBwtP4r5m84ts5lV3FISUjAhawI+LPwQpadKfdsDSYxPlCwjDYhQWW5uDBsmdbqefNK5TajckrW13Nzmm7y4JZ3m2eyOA4RW3JyulVfLTX3eTdy8XM/kZPMYXsXNCeu1drsO4YKIXiaiTUS0jog+MtIqqn1PGwWnNxPRdfXZDy1uGh8jMkegS1oXv5I0tSEw5D8nOwenK09j9vrZACI/rxYq6mq5vfgikJNT8zEuv9xdJLwISajm3LzO69TGcquudm5j7Yfbg0Jt5/icqK3l5uWBAnC+nkTmPi/i5sUVHPh7BPkcwAXMPADAFgBPAwAR9QMwAUAWpNj0X4nIJmtoaNDipvERHxePSQMn4bNtn2HvcZfJHgcC59Yu6XQJzm99PooOS7ryxDiXO0IDoq6W21NPATNm1L0foZpzs/bV6cnf6hLzsvbOrU+qntoZF+93KC232kbA1tUt6cVyA8zgJS/i5mYtR5vlxsyfMbMqpLkCZkWXmwG8y8zlzLwdkif4YrtjhAJq7Ckau3Tpwm+/Xf81yxoLJadKMGnVJEw+bzLu7Fq7tWUbj27ElLwpeKn/S7i4lfzNztk1B//Y/g/EIQ7/ufI/9dHlsKGqMAcWjKxtm1CxZ4+sgWrTxjnvYFmZZPno0cP5ps3sXAzTipexbdsm52zb1r6em6KkRATCKYNHRYWZdmrgQGcxKSqSTCDt2jlXGCgvN5MdO/W9utqsdu12vs2bZZ1berpzaZnTp82F3tnZ9hUNAKmyXVkp352TeO3dKwvQW7WyL8MTeD5VwbuujBgxogLAesum6cw8vbbHIaKPAbzHzLOI6HUAK5h5lrHvTQCLmPmDuvfYBmZu1D9NmzZlTe247M3LuO/rfbm6urpWn1u2fRnjOfDS4qW+bSVHS5ieI27yuyah7mbYSUpiBtzbiFSEpz/Tpsm5HnrIuc2iRdJm6VLnNtXV3vrtpc0dd0ibKVPc29XEnj3m+U6ccG43bpy0eewx5zZVVTX3vaLCbHPmjHO7K66QNjff7Nxm82bzWCdPOre76ippM3u2c5vnn5c2Dzzg3KagwDzfunXO7WoDgJPscl8FsATABpufmy1tpkFqcyoj6nUAd1v2vwlgnNt56vKj3ZKaIHIH5qKwtBCr966u1efsQv47p3XGyB4j0SSx4Ydx5ecDM2fW3C5cKHeUk2UASLLoF1+U+TsnvFRNqG2f6jr349Ut6WXOLS5OinoOG+bcJjFRrmOTJu6uvdrOubkda8gQeS0tde9X4DEDiYRbkmsoRk1EuQBGA5hoiCUgBaat9nW9Fp128RxrYpXbs27HI58+ghl5M3BRp4s8f84p5P/1G17HptJNIe1jJOjTx6x87MTWrWYZk/pG3dTcZhZSUmSOLxTcfbf70gRrn0Ipbm4L69V5arqpFxTULOJNm7rPfwHeoiWtQuT24PHss+LivOce5zanT8urm8BH25wbEV0P4EkAVzKztZbWAgBziOgVAB0B9AKwsr76ocVNE0R6Sjpu6XML3tnwDl657hXX1FfbDm/Dlzu/xH2D7nNcrN2rdS/0at2rXvscLfTs6TwXE2rCHRnnZeo6VOLmFohhRYlHTTd1L9ZpkybuwSRA7QNK3EhLA/72N/c2pwxp8BIxCkRNtOTrAJIBfE5y4Vcw84PMvJGI3gdQAKASwMPMRqb1eiBSlbjHE9FGIqomoiE2+7sS0QkiesKy7XpjbUQREYXoWVTjRG62FAr9eMvHru1m5s/E/Qvux8aDGx3FTVM/uN1gI4USGze3nRe8jk2JVihu6k2bhkbcQikwXsQt2iw3Zu7JzF3YLDj9oGXf8ywFp89n5kX12Y9IzbltAHArgK8c9r8CwDdwYy3EXwDcAKAfgDuNNROaeuKa865Bp+adakzHpbKZzMyfGVSUVFO/qBt7NAY8h8uCUNcgFDf12oibm3iHcg7ziSdkvtTNdRlt4hYtRETcmLmQmTfb7SOisQC2A9ho2XwxgCJmLmbmCgDvQtZMaOoJVSj006JPsf/Efsd2ylqbtW4WTp2Vx0wtbuEhlDfRUFuB4XaPhaL/zz0nYuLlPOGymrt2lUw1DWmdW7QQVdGSRNQMwC8A/DpgVycAJZb3u41tTseZTESriWh1ZWWlUzNNDeRk56CKqzB73WzHNkrc9p3Yh0+2fgLAvyippmFQVgYcP1734yjB9VJdIRSo84TCeh0/Hrj6avc2XgJKwo01aMUtgCXWqLc/QSJaQkQbbH7cLK7nAPyJmU/U5dzMPJ2ZhzDzkIRo+itsYPRp0weXdLrEtc5bRVUF2jZti9ZNWmNRkXiSteUWXkJxY09NrTlS0AuhtCZrcz63VF6hJNyWmxes1zzc1z+aqbeviJmvPYePXQJgHBH9HkALANVEdAbAGoRxfYTGJDc7Fz/75Gf4Yd8PuLBjcHqHiqoKNEtqhtG9R+O1la8B0OIWLqL5RhauecBwzzt6mXPTRAdR5ZZk5uHMnMnMmQD+DOAFZn4dwCoAvYjoPCJKgiTfXBDBrsYMd2TdgeT4ZMzMt1+9XFFV4avfptDiFl6iMaAkVLRr561duMWtJsvtD38A/vzn+u+PxplILQW4hYh2A7gUwCdEtNitPUsSzikAFgMoBPA+M290+4wmNLRs0hI397kZs9fP9kVDWimvKkdyQjIGtR+E/u36I47iEB+nHf/hIBqjJUNpTR49ChQXu7cJ5ZybF7yK2+OPA48+Wv/90TgTqWjJj5i5MzMnM3MGMwfV9WHm55j5D5b3C5m5t7FG4vnw9ji2yR2Yi8OnD/sCRqwoy42IMG34NNzU+6YI9DA2iUa3ZCgFNy2t5gXR4RZ4db5omnPT2BNVbklNdDKyx0h0aNbB1jWpxA0A7rjgDsybMC/c3YtZso0ar1ddFdFuRJTcXHn9yU/Ce1495xb96OcPTY0kxCXg7gF3408r/oSDJw+iXao5EWIVN014ufRS4OBBKS8TLYTbkho8ODJuWW25RT/actN4ImdgDiqrKzFn/Ry/7VrcIks0CVssoIRUryeLfrS4aTyR1S4LQzoOCUrHpcVNY+UXvwDGjAHuvTfSPalfwrVIXXPu6K9I45ncgbnIP5CPvP15vm0VVRU6I4nGR0YGMH9+zTkaGyrK7RptwTz33Qc8+GDN7WIJLW4az0y4YAKS4pMwM88MLCmvLNeWm0YTYd58s+byObGGFjeNZ1o3bY2bet+EWetn+XJKarekJpaIpjWFGne0uGlqRW52LkpPlWLRVskjqcVNo9FEI1rcNLXiuh7XISM1w7fmTYubRqOJRrS4aWpFYnwiJvafiI+3fIwfT/6oA0o0Gk1UosVNU2tysmXN2zsb3tGWm0ajiUq0uGlqzYCMARjcYTBm5M3Q4qaJSaJtKYAmGC1umnMiZ2AO1u5fCwZrcdNoNFGHFjfNOXFX/7uQGCfZY7W4aWINvSQg+tHipjkn2jRtg9G9RwMAkhN0QIlGo4kutLhpzpmcgTkA4LPgNJrGTrLxHJeknRVRT6QqcY8noo1EVE1EQwL2DSCi74z964koxdh+ofG+iIheJdJTupHmxl434pnLn8GNvW6MdFc0mrDw3/8NTJ3a+BNDNwaII+A8JqK+AKoB/B3AE8y82tieAOAHAPcwcz4RtQZQxsxVRLQSwCMAvgewEMCrzLyopnOlpqbyyZMn62soGo1G0+ggolPMXEMd9OgmIpYbMxcy82abXaMArGPmfKPdIUPYOgBIY+YVLGr8FoCxYeyyRqPRaBoQ0Tbn1hsAE9FiIvqBiJ40tncCsNvSbrexzRYimkxEq4lodWVlZT12V6PRaDTRSL0VSyeiJQDa2+yaxszzXfpzOYCLAJwC8B8iWgPgaG3OzczTAUwHxC1Zm89qNBqNpuFTb+LGzNeew8d2A/iKmUsBgIgWAhgMYBaAzpZ2nQHsqXMnNRqNRtMoiTa35GIA/YmoqRFcciWAAmbeB+AYEQ01oiQnAXCy/jQajUYT40RqKcAtRLQbwKUAPiGixQDAzEcAvAJgFYA8AD8w8yfGxx4C8AaAIgDbANQYKanRaDSa2CQiSwHCiV4KoNFoNLVDLwXQaDQajSYKafSWGxFVAzh9jh9PABBrawn0mGMDPebY4FzH3ISZG7Tx0+jFrS4Q0WpmHlJzy8aDHnNsoMccG8TimBUNWpk1Go1Go7FDi5tGo9FoGh1a3NyZHukORAA95thAjzk2iMUxA9BzbhqNRqNphGjLTaPRaDSNDi1uGo1Go2l0aHEDQETXE9Fmo8r3Uzb7k4noPWP/90SUGf5ehg4P473CKDlUSUTjItHHUONhzI8RUQERrSOi/xBRt0j0M9R4GPeDRoX7PCL6hoj6RaKfoaSmMVva3UZETEQNOlTew3ecS0Q/Gt9xHhH9NBL9DDvMHNM/AOIhuSq7A0gCkA+gX0CbhwD8P+P3CQDei3S/63m8mQAGQIrCjot0n8M05hEAmhq//6whf8e1HHea5fcxAD6NdL/re8xGu+YAvgKwAsCQSPe7nr/jXACvR7qv4f7RlhtwMYAiZi5m5goA7wK4OaDNzQBmGr9/AOAaozpBQ6TG8TLzDmZeB6A6Eh2sB7yMeRkznzLeroB/iaWGipdxH7O8TQXQ0CPMvPw/A8BvAbwE4Ew4O1cPeB1vzKHFTSp6l1je21X59rVh5kpI8dTWYeld6PEy3sZGbcd8PxpH1QlP4yaih4loG4DfA3gkTH2rL2ocMxENBtCFzYojDRmvf9u3GS73D4ioS3i6Flm0uGk0FojobgBDALwc6b6EC2b+CzP3APALAL+MdH/qEyKKg5TVejzSfQkjHwPIZOYBAD6H6YVq1Ghxk4re1icZuyrfvjZGEdV0AIfC0rvQ42W8jQ1PYyaiawFMAzCGmcvD1Lf6pLbf9bsAxtZrj+qfmsbcHMAFAL4goh0AhgJY0ICDSmr8jpn5kOXv+Q0AF4apbxFFi5sURu1FROcRURIkYGRBQJsFAHKM38cBWMrGTG0DxMt4Gxs1jpmIBgH4O0TYDkagj/WBl3H3srz9CYCtYexffeA6ZmY+ysxtmDmTmTMh86tjmHl1ZLpbZ7x8xx0sb8cAKAxj/yJGQqQ7EGmYuZKIpgBYDIk8+iczbySi3wBYzcwLALwJ4G0iKgJwGPIH1CDxMl4iugjARwBaAriJiH7NzFkR7Had8PgdvwygGYC5RqzQLmYeE7FOhwCP455iWKxnARyB+RDXIPE45kaDx/E+QkRjIKVvDkOiJxs9Ov2WRqPRaBod2i2p0Wg0mkaHFjeNRqPRNDq0uGk0Go2m0aHFTaPRaDSNDi1uGo1Go2l0aHHTRB1EVGXJYJ7X0KswKCzZ2d+o43GeI6InLO+HEtE/6t7DWvfjRA37mxjfXwURtQlXvzQaQK9z00Qnp5k5226HkbCamLmhJnV+j5mnBG4kogQjb+m5cAOAT+vWrdDDzKcBZBuZQDSasKItN03UQ0SZRr2qtwBsANCFiKYS0SojGeyvLW2nEdEWozbZO8rCIaIvVIolImqjbrhEFE9EL1uO9YCx/SrjMx8Q0SYimq0qQRDRRUS0nIjyiWglETUnoq+IKNvSj2+IaGAN48ologVEtBTAf4ioGUktuR9IaqzdbGnrGxeA8wMOdQ2AJUSUZfQnzxhLL+Oz84hoDRFtJKLJlmOeMMa+kYiWENHFxpiLjUW/qo/zje1biehXDmOx/T40mkihLTdNNNKEiPKM37cD+L8AegHIYeYVRDTKeH8xAILkBrwCwElI9phsyN/2DwDW1HCu+wEcZeaLiCgZwLdE9JmxbxCALAB7AXwLYBgRrQTwHoA7mHkVEaUBOA3JYpML4L+IqDeAFGbO9zDWwQAGMPNhkryltzDzMcONt4KIFhhtbMdltDvLzEeJ6HcA/peZZxupmOKNc9xnHL8JgFVE9G9mPgQpcbOUmacS0UcAfgdgJIB+kOS6KpvHxZB8jKeMz39iTVfl9H0w81cexq/R1Ata3DTRiJ9b0phz28nMK4xNo4yftcb7ZpCba3MAH6m6bIYw1MQoAAPIrDiebhyrAsBKZt5tHCsPUsT1KIB9zLwKMOuhEdFcAM8S0VQA9wGY4XGsnzPzYTVUAC8YQl0NKV2SAWC4y7hGAVBi/B2AaUTUGcCHzKzyRD5CRLcYv3cxxnfIGKNyZ64HUM7MZ4lovTFWax8PGef+EMDlAKy5GJ2+Dy1umoihxU3TUDhp+Z0AvMjMf7c2IKL/cvl8JUw3fErAsX7OzIsDjnUVAGtlgCq4/L8w8yki+hxSKPJ2eM+8bh3XRABtAVxoiMyOgL7acQOkhAuYeQ4RfQ9JgLzQcLFWA7gWwKVGH7+wHPOsJQF4NYzxMnO1YUX6hhc43ID3tt+HRhNJ9JybpiGyGMB9RNQMAIioExG1g1gKY40oveYAbrJ8ZgdMwRkXcKyfEVGicazeRJTqcu7NADqQJJeGMd+mhOANAK8CWMXMR85hXOkADhrCNgJAN2O77biMOcABAPKM990BFDPzqwDmG/vSARwxhK0PpMRLbRlJRK0Mt+ZYiIvWitP3odFEDG25aRoczPwZEfUF8J0R43ECwN3M/AMRvQcgH8BBSDkQxR8AvG8EVFgrML8BccH9YIjFj3CpacbMFUR0B4DXjJv9aYhldIKZ1xDRMQD/OsehzQbwseEWXA1gk3FOp3FdCGCtxfq6HcA9RHQWwH4AL0AswweJqBAizMq1WxtWAvg3pFbYrMDyME7fh9FXjSYi6KoAmkYLET0HEZ0/hOl8HQF8AaCP3VIFIsoFMMRuKcA5nu+XAIqY+d1QHM/hHLmoY58N9+oQZi4NVb80mprQbkmNJgQQ0SQA3wOY5rIG7zSAG6iOi7gVzPy7+hS2umK4UfMAJELm9DSasKEtN41Go9E0OrTlptFoNJpGhxY3jUaj0TQ6tLhpNBqNptGhxU2j0Wg0jQ4tbhqNRqNpdPx/G4lXtvvjmj4AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "fig, ax1 = plt.subplots()\n", + "ax1.set_title('Digital filter frequency response')\n", + "\n", + "ax1.plot(w / (2 * np.pi), 20 * np.log10(abs(h)), 'b')\n", + "ax1.set_ylabel('Amplitude [dB]', color='b')\n", + "ax1.set_xlabel('Frequency [rad/sample]')\n", + "\n", + "ax2 = ax1.twinx()\n", + "angles = np.unwrap(np.angle(h))\n", + "ax2.plot(w / (2 * np.pi), angles, 'g')\n", + "ax2.set_ylabel('Angle (radians)', color='g')\n", + "ax2.grid()\n", + "ax2.axis('tight')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(63,)" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "b.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [ + "def optimfuncQMF(x):\n", + " \"\"\"Optimization function for a PQMF Filterbank\n", + " x: coefficients to optimize (first half of prototype h because of symmetry)\n", + " err: resulting total error\n", + " \"\"\"\n", + " N = 2 #4 subbands\n", + " cutoff = 1.5 #1.5\n", + " h = np.append(x, np.flipud(x))\n", + " f, H_im = sig.freqz(h)\n", + " H = np.abs(H_im) #only keeping the real part\n", + " \n", + " posfreq = np.square(H[0:512//N])\n", + " \n", + " #Negative frequencies are symmetric around 0:\n", + " negfreq = np.flipud(np.square(H[0:512//N]))\n", + " \n", + " #Sum of magnitude squared frequency responses should be closed to unity (or N)\n", + " unitycond = np.sum(np.abs(posfreq + negfreq - 2*(N*N)*np.ones(512//N)))//512\n", + " \n", + " #plt.plot(posfreq+negfreq)\n", + " \n", + " #High attenuation after the next subband:\n", + " att = np.sum(np.abs(H[int(cutoff*512//N):]))//512\n", + " \n", + " #Total (weighted) error:\n", + " err = unitycond + 100*att\n", + " return err" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "metadata": { + "Collapsed": "false" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.0\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nO3dd3xUVfr48c+TRgglhSQQEkLvHQMIoqCAYEVX17Wsoqvr6vbid7/6dVdd19113aJbv/74KnaxF0SFRQTpJbQQauikkUYSEtLz/P6YC44xIT035Xm/XvPK3Dvnzjz3zmSeOefce46oKsYYY0x9+bgdgDHGmLbJEogxxpgGsQRijDGmQSyBGGOMaRBLIMYYYxrEEogxxpgGsQRiWpyIrBKRe87z+LMi8uuWjKmtEJFYESkQEd8Gbn9URGY1dVytiYjsFpEZbsfREVgC6eCcL5Qi50spXUReFJGuLfj6d4rIWu91qnqfqv62GV6rn4ios68Fzr4/6PW4iMh/iUiSc0yOi8jvRSTAq8yLznPMq/LcTzvr7/Tarwqv1yoQkX9WE9MtIrK3yrrlNax7UFWPq2pXVa1oosPSpETkMREpq7Lfv2zG13tRRJ7wXqeqI1V1VXO9pvmSJRADcI2qdgXGAeOBh1yOp7mFOPt7C/CIiMx11v8duBe4A+gGXAHMAt6osv0BpwwAIuIH3AQcqlJug/Nlf/b2w2piWQ0ME5EIr+caC3Susm6KU7YteLPKfj/ldkCmeVgCMeeoajqwDE8iAUBELhSR9SKSKyI7vZsGnF/Zh0XktIgcEZHbnPWPicirXuXO/vL38349ERkOPAtMcX6p5jrrz/2qFJEZIpIsIr8QkQwRSRORu7yeo4eIfCQi+SKyRUSeqFqjOc/+bgB2A6NEZDDwfeA2Vd2gquWquhu4AbhKRKZ7bfoRME1EQp3luUACkF6X160SQwpwGLjEWTXBiemLKut8gC1Vj6XTHPhbEVnnvA//EZFwr+Nzu4gcE5FsEXnY+7VFpJOIPCMiqc7tGRHp5Dz2hYjc4Ny/yHnNq5zlmSKyoz77Wdtnog77Mc3rc3jC+ezdC9wG/NL5/HzklD3XTFfLPp73s2VqZwnEnCMiMXh+dR90lqOBj4EngDDgAeBdEYkQkS54frFfoardgKlAvb5UVHUvcB9f/lIPqaFoLyAYiAbuBv7l9eX9L6DQKTPfudVlX0VELgJGAtuBmUCyqm6uEuMJYCNwudfqYuBD4GZn+Q7g5bq8bg1W82WyuARYA6ytsm6jqpbVsP2twF1AJBCA531CREYA/wvcDvQGegAxXts9DFyI5wfDWGAS8CvnsS+AGc796Xw1yU13Hm9qNe1HX+BT4B9AhBPvDlVdALwGPOV8fq6p5jnPt49w/s+WqYUlEAPwgYicBk4AGcCjzvpvA5+o6ieqWqmqy4F44Ern8Uo8v947q2qa84u9OZQBj6tqmap+AhQAQ8XTkXwD8KiqnlHVPcBLdXi+LCAHeA54UFVXAOFAWg3l0/B8cXl7GbhDRELwfKF+UM12Fzq/mM/eLqzh+b1rGxfjSSBrqqw73xf2C6p6QFWLgLf4sgZ5I7BEVVeragnwazzv2Vm34TmuGaqaCfwGT7I5G9PZWtclwB+8lmtLIDdV2e/e5ylbl/24FfhMVRc5n4FsVa3rj5Xz7SPU8Nmq43N3eJZADMB1Ti1iBjAMz5cpQF/gm95fBsA0IEpVC4Fv4alBpInIxyIyrJniy1bVcq/lM0BXPF/qfngS31ne92sSrqqhqjpcVf/urMsComooH+U8fo6qrnVe/2E8X9JF1Wy3UVVDvG4ba3j+1cAY55fvhXhqZPuAKGfdNM7f/+HddHb22ICn1nHueDjvWbZX2d7AMa/lY846gA3AEBHpieeL/GWgj9OsNKmWeN6qst+p5ylbl/3ow9f7l+rqfPsINX+2TB1YAjHnqOoXwIvAn51VJ4BXqnwZdFHVJ53yy1R1Np4v2H3A/znbFQJBXk/d63wv24iQM4Fyvtos06eBz/U5ni/ISd4rRaQPni/1VdVs8yrwCxrXfIWqHgZS8XTgH1fVAuehDc66rnia0eorDa/jISJBeJqxzkrF8yPhrFhnHap6BtgK/ARIVNVSYD3wc+CQqn4lodZBfT4TVZ0ABtbwWG2fnxr30TSeJRBT1TPAbBEZi+cL8hoRmSMiviIS6HQ8xohITxGZ5/SFlOCp+p9tHtkBXCKeaxaCOf9ZXSeBGPE6VbaunFNZ3wMeE5EgpwZ0Ry2b1fRcB/B06L8mnhMHfEVkJPAuni/Oz6rZ7O/AbJrm7Kg1eL6c13itW+usi6+hhlObd4CrnQ7oAOBxvvo/vwj4ldOnFQ48guc9P+sL4Id82Vy1qspyfdTnM1HVa8AsEblJRPzEc+LE2eatk8CA82xb2z6aRrAEYr7CaSd+GXjE6UCeB/wPnl/7J4D/wvO58cHz5ZaKpz9hOnC/8xzLgTfxnJm0FVhynpf8HM9ZR+kiUt9fteD5QgvG0/zxCp4vjJIGPM/Z53oOzxfMGSART5PHdapaWbWwquao6gptmkl1vsDTeex9BtkaZ12DEpTTJ/UD4HU8tZFTQLJXkSfw9GklALuAbc4675i6eb1+1eX6xFKfz0TVbY/j6Xf7BZ7P2g48HeIAzwMjnCbW6vqhattH0whiE0qZ9kRE/gj0UtU6nY1Vy3P9BrgeuERVcxsdnDHtjNVATJsmIsNEZIxzWu4kPKdivt8Uz62qjwIL8PSBGGOqsBqIadNEZCKeZqveeNrDFwBPNlGzkjHmPCyBGGOMaRBrwjLGGNMgfrUXaT/Cw8O1X79+bodhjDFtytatW7NUtepoDB0rgfTr14/4+Hi3wzDGmDZFRI5Vt96asIwxxjSIJRBjjDENYgnEGGNMg1gCMcYY0yCWQIwxxjSIJRBjjDENYgnEGGNMg1gCaSGf7krjUGZB7QWNMcZRWaks2nycnMJSt0OpliWQesg8XcI/ViRRWFJee2Ev6w5mcf9r23hscXNNGW6MaY9WJ2Xy0Hu7eODtndR33MLP9pxk2e702gs2giWQevj1B4n8ZfkB/rXyYJ23KSgp55fvJOAjsCYpi+PZZ5oxQmNMe7Jo83F8BD7fl8G721LqvF1KbhE/XLSNH72+vVlbPiyB1NHn+06ydHc6Ed068dyaI3VOBH/4ZC+peUX845YJ+Ai8seV4M0dqjGkPMvKL+WxvBndP68/EfqH85qPdpOcV12nbJz/dhyp08vfh1x8k1rv2UleWQOqgqLSCRz7czaDIrrx3/1T8fIXffbLnvNsUl1Xwt8+SeG3Tce6+qD9XjYnismE9eSs+mbKKr82OiqqycO0RLvvLKo5mFTbXrhhjWomH39/Fbc9tJON09Unh7a3JVFQqt0yK5akbx1JWUcm9r8SzP/30eZ9385EcPtqZyvemD+SXc4ex/lA2i3emNscuWAKpi398nkTyqSKeuG4UfcKC+MGlg1i2+ySrD2R+rWxeURkfbE9h7jOrefqzA1w1JooH5gwF4NbJfcgqKGHF3pNf2aasopL/eT+Rx5fs4XBmIb/+8Ku/GMqrSTjGmLbF+/941f4MXtt0nHUHs7nun+vYk5r/lbJnO8+nDOjBgIiu9A/vwtM3jeN4zhmu/PsaHv9oD/vS879WsyirqOTxJbuJCg7k/ukDuXVSLGNjgvntkj3kFZU1+T65OhqviMwF/gb4As+p6pNVHu8EvAxcAGQD31LVo85jD+GZvrQC+LGqLmuuOCsqlZviYrhwQA8A7p7Wn3e2JvP917ax4I4LmDownMSUPJ5atp/1B7Mor1QGhHfh5e9M4pIhX46APH1IJFHBgfzj84McyiykslLZd/I0246dIi2vmO/PGEh41048vmQPSxLSmD40gh8v2s7BjAL+87NLCAqo/e3al57PvrTTXDc+urkOhzEd3omcM6xJyuLWybF1Kv/vVQd5dtUh/nbLeKYM6MEjH+5mQEQX/vzNsXz/1W1c9+91jOsTwtiYYEK7BJCRX0LyqSJ+OXfYuee4YnQUkwf04E/L9vHC+iMsXHeEmNDO3DGlL3dd1J/yCuX+17aSmJLPv26dQOcAXwB+d/1o/uudBLIKSgju7N+kx8G1GQlFxBc4AMwGkoEtwC2quserzPeBMap6n4jcDFyvqt8SkRF4pjGdhGcq08+AIapacb7XjIuL04YO566qiMi55fS8Ym5/fhPHss8wZ1QvPk5IJaxLAN+M68Os4ZGM6xOKr4987XmeW3OYJz7ee245OqQzY/sEc+3Y3swdFUVFpXLdv9aRnl9McGd/jmQVUlGp/M+Vw7j3koG1xnn785tYfyib7Y/Mpntg035YjDEeD723i0Wbj7P0pxczrFf385bNO1PGtD9+TlFZBZWqTOofxsbDObx+z2SmDgonI7+Y//3iENuP57InLZ/Sck9NJTYsiOU/v4ROfr5fe86M/GJW7MtgSUIq6w5mM6RnV4IC/EhIzuV314/mlklfTWxVv7/qS0S2qmpc1fVu1kAmAQdV9TCAiLwBzAO8OxfmAY85998B/imeozAPeENVS4AjInLQeb4NzRVs1YPfKziQt++bwl0vbmFJQiq3X9iXX1w+tNYMf8/FA7hzaj8qVFGFQP+vfjh8fYTfXT+Kef9aR2l5Ja/ePZl/rzrIgtWHuf3Cfud+VVQn43Qx6w5mUamw4VA2c0b2avgOG2NqtPagp/n6wx2pDJt7/gTy/LojnC4p5937p/LsF4dYvuck14+PZuqgcAAiuwfy6DUjAU8zV3ml4usj+PlIjV/6kd0DuWVSLLdMimX5npM8tng3R7PO8K9bJ3DF6KivlW9M8jgfNxNINHDCazkZmFxTGVUtF5E8oIezfmOVbVu8zSYkKIA3751CZkEJ0SGd67ydn6/PeQ/8mJgQXr/nQmJCO9MnLAh/X+HGZzfw2qZj3HPxgHPlSsorOJhRwMjewQB8nJBGpYK/r7DuYJYlEGOawfHsM5zIKcLfV1i8I5X/unwoPj5CVkEJRaUV9AkLOlc2r6iMF9YdYc7InlzQN5T/9+0L+M+ek0wbHF7tc/v5+lBNheO8Zo/oycWDwzl1ppSo4Lp/DzWFdt+JLiL3iki8iMRnZn6907uxAvx86pU86mrKwB7nPohx/cK4aFAPnv3i8LmLGHPPlHL785u56u9rWZLgOcPiwx2pjIjqzrRB4axNymrymIwxsMapfXzvkoGk5Bax7fgpikor+OazG5j99Bf8x+vivefXHuF0cTk/njkYAB8fYe6oXnTt1LS/3QP9fVs8eYC7NZAUoI/XcoyzrroyySLiBwTj6Uyvy7YAqOoCYAF4+kCaJHIX/GTmEG76fxuY/qeV3DIplk92pXEip4i+PYL49QeJ9OoeyI4TuTx0xTD8fH1YuX8PKblFzZLcjOnI1iZlERUcyH0zBvLc2sN8sCOFj3elcSSrkEGRXfneq1v53iUD2Xkilw2Hs5kzsue5VoL2xs0ayBZgsIj0F5EA4GZgcZUyi4H5zv0bgc/V0+u/GLhZRDqJSH9gMLC5heJ2xaT+YSz67oWMiQnhH58fJKuglFfunsTz8+MoLK3gzhe2IALXjuvNxU71eG1S09e4jOnIKiqV9YeymTYonK6d/Jg1vCfvbUvhhXVHmT+lLx/9cBqzhvfk2S8OcTS7kAevGMZfbhrndtjNxrUaiNOn8UNgGZ7TeBeq6m4ReRyIV9XFwPPAK04neQ6eJINT7i08He7lwA9qOwOrPZgysAdTBvbgWHYhAX4+56qsD1w+hN9/so/J/cOICu5Mr+5KZLdOrD2Yzbcm1u00Q2NM7RJT8sgrKjvXhzFvXDRLEtLo1yOI/75iGJ0DfHn22xeQkJzLqOhg/H3bdy+Bq9eBqOonwCdV1j3idb8Y+GYN2/4O+F2zBthK9e3R5SvLd08bQGpuMZeP6Al4zriYNiicVQcyqaxUfKo5ndgYU39rD3r6Fi9yzqCaPiSCm+Ji+PaFfc9dp+XrI4yPDXUtxpbUvtNjB+HrIzx27chzpwUCTBscTk5hKTuTc12MzJj2Q1VZuS+D4VHdCe/aCfCcRPPUjWMZExPicnTusATSTs0YGklIkD8/eWNHjWPtGGPq7unlB4g/dopv2CgP51gCaafCugTwwp0TyTxdwp0Lt5Bf3PTj4BjTUby0/ih///wgN8XFcM/F/d0Op9WwBNKOjY8N5dnbL+DAydP86v1Et8Mxpk3acSKXxz7azewRPfn99aOb7arutsgSSDs3fUgEt0/py9LEdKuFGNMA725NppOfD09/axx+7fysqvqyo9EBXDu2N6UVlfxn98naCxtjzimvqOSTXWnMHN6zya8ebw8sgXQA4/qEEBPamY+8JpU5klVIcVm7v3TGmHo7mFFwbtK3DYezyS4s5ZoxvV2OqnWyBNIBiAjXjO3NuoNZ5BSWsu34KWb/9Qu++3I8lZVtdnQXY5rcqv0ZzPrrF/zirZ2oKkt2ptG1kx8zhkbUvnEHZAmkg7h6TBTllcpb8Sf40evb6eTnw5qkLBauO+J2aMa0CpmnS3jg7Z10CfBl8c5UXt98nE8T07h8RM+vTbtgPCyBdBAjorozIKILT366j5P5xbxyz2Rmj+jJH5fuIzElz+3wjHFVZaXywNs7OV1czjv3T2XKgB48/H4i+cXlXDPWmq9qYgmkgxCRc+24D8wZyoTYUP54wxjCugTwwNs7vza3sjEdyXvbU/jiQCa/umo4w6O688zN4wjrEkBIkP+5YUvM19lpBR3I3Rf3p194EPPGeq6kDesSwM9mDeHB93axKyWvww7HYMxbW04wKLIr376wLwA9uwfy6t2TKSgpJ8DPfmfXxI5MB9I90J/rx8d8ZXDFK0ZF4e8rfLgj9TxbGtN+peQWsfloDvPG9v7KRYIjendnUv8wFyNr/SyBdHDBQf7MGBrJkoRUKuyMLNMBLXFOb792nPV11JclEMO1Y3tzMr+EzUdy3A7FmBa3eGcqY/uEfG2aBFM7SyCGWcN7EuScupieV8z8hZt5Ysket8MyplnsSc3n8qe/4P3tyRzMKGB3aj7X2plWDWKd6IbOAb7MHtGTJQmpLN+TTlZBKauTMvnGhBhG9O7udnjGNKnff7KXAycL+NmbOxkQ3gURz3VSpv6sBmIAmDeuN6eLywkJCuDd+6fQPdCfp5btczssY5rUmqRM1h7M4n+uHMb3LhnA4axCpgzoQc/ugW6H1iZZDcQAcOnQSF68ayIT+4XRpZMf358xkD98uo8Nh7KZMrCH2+EZ02iVlcofl+4jOqQz86f2o5OfL3NG9aKXJY8Gc6UGIiJhIrJcRJKcv9VOICwi850ySSIy31kXJCIfi8g+EdktIk+2bPTtk4gwY2gkXZwRR+dP7UdUcCBPLt1n42WZduHjXWkkpuTz89lD6OTnGZpkQmwovUM6uxxZ2+VWE9aDwApVHQyscJa/QkTCgEeBycAk4FGvRPNnVR0GjAcuEpErWibsjiPQ35efzx7CzhO5vLj+qNvhGNMoWQUl/OajPQyP6s51NiVtk3ErgcwDXnLuvwRcV02ZOcByVc1R1VPAcmCuqp5R1ZUAqloKbANiWiDmDufGC2KYNTySJz/dx960fLfDMaZBVJVfvpNAfnEZf71pLL4+NqNgU3ErgfRU1TTnfjrQs5oy0cAJr+VkZ905IhICXIOnFlMtEblXROJFJD4zM7NxUXcwIsIfbxhDcJA/P1603eYPMW3SyxuO8fm+DB66YhjDo+yswqbUbAlERD4TkcRqbvO8y6lnFL96N7KLiB+wCPi7qh6uqZyqLlDVOFWNi4iwMf3rq0fXTvz1prEkZRSwYHWNh9mYVimnsJTff7KXGUMjuHNqP7fDaXea7SwsVZ1V02MiclJEolQ1TUSigIxqiqUAM7yWY4BVXssLgCRVfaYJwjXncfHgCKYNCuedrcn86LJBXxkvyJjWbElCKiXllfz33GH2uW0GbjVhLQbmO/fnAx9WU2YZcLmIhDqd55c76xCRJ4Bg4KctEKsBrh8fzfGcM2w7fsrtUIyps/e2pTCsVzdrumombiWQJ4HZIpIEzHKWEZE4EXkOQFVzgN8CW5zb46qaIyIxwMPACGCbiOwQkXvc2ImOZM6oXgT6+/DethS3QzGmTg5nFrDjRC7fmGBnXTUXVy4kVNVsYGY16+OBe7yWFwILq5RJBqwu2sK6dvJjzsheLElI45FrRpw7j96Y1uqD7Sn4CMwbZwmkudhQJqbOrh8fTV5RGSv32dlspnVTVd7fkcJFg8JtmJJmZAnE1Nm0QeGEd+3E+9uT3Q7FmPOKP3aKEzlFXG8XDTYrSyCmzvx8fbhmbBQr92dSUFLudjjG1GjJzlQC/X2YM7KX26G0a5ZATL1cOTqK0vJKVu6r7sxrY9xXWaks232S6UMizo3tZpqHJRBTLxNiQwnv2omlu9PdDsWYau1MziU9v5i5o6z20dwsgZh68fURLh/Zk5X7MmxoE9MqLU1Mx99XuGxYdSMkmaZkCcTU2xWjenGmtII1SVluh2LMV6gqS3enM3VgOMGd/d0Op92zBGLq7cIBPege6MfSRGvGMq3LvvTTHMs+Y81XLcQSiKk3f18fZo3oyWd7T1JWUel2OMac82liOiIwe4Q1X7UESyCmQa4YFUVeURnffm4TH+1MpbTcEolxz8GM0/zmo90sXHuEif3CCO/aye2QOgRLIKZBZg6L5OErh5OSW8SPFm3ngbd3uh2S6aC2Hz/FnGfW8OrGY8wYGsGT3xjtdkgdhiUQ0yA+PsJ3LxnA6v+6lDum9OXjXWlk5Be7HZbpgF7ZcIwgf1/WPXgZ/7x1AgMiurodUodhCcQ0io+PcNdF/amoVN7eakOcmJaVd6aMj3elMW98byK72ZhXLc0SiGm0/uFdmNw/jLfiT1BZWe/JJY1psA93plBSXsnNE2PdDqVDsgRimsTNk/pwLPsMGw9nux2K6SBUlUWbTzCyd3dGRQe7HU6HZAnENIkrRkXRPdCPRVtOuB2K6SASkvPYm5bPzZOs9uEWG2nMNIlAf1+uHx/Nq5uOU1BcxsWDI7hlUiydA2ziKdO0Nh/JYfHOFFbszSDQ34d543q7HVKHZQnENJmfzR6CiLD6QCYr9+/hWHYhv5k3yu2wTDtyNKuQmxdsINDfl6kDe3Db5L50D7QhS9xiCcQ0mZCgAB67diQAP3ljO+9tS+HBK4ZbLcQ0mTe2nEBE+PwXM+gVbGdduc21PhARCROR5SKS5PwNraHcfKdMkojMr+bxxSKS2PwRm/q4dVIsp0vK+Sgh1e1QTDtRWl7JO1tPcNmwSEserYSbnegPAitUdTCwwln+ChEJAx4FJgOTgEe9E42IfAMoaJlwTX1M6h/GoMiuLNp83O1QTDuxfM9JsgpKuXWydZq3Fm4mkHnAS879l4DrqikzB1iuqjmqegpYDswFEJGuwM+BJ1ogVlNPIsItk2LZfjyXvWn5bodj2oHXNx8jOqQzlwyOcDsU43AzgfRU1TTnfjpQ3fCZ0YD3eaHJzjqA3wJ/Ac6c70VE5F4RiReR+MzMzEaGbOrjhgnRBPj58Pomq4WYxjmaVci6g9ncPLEPvj7idjjG0awJREQ+E5HEam7zvMupqgJ1voRZRMYBA1X1/drKquoCVY1T1biICPvl0pJCggK4Zkxv3txygs1HctwOx7RRpeWVPPTeLgJ8fbhpYh+3wzFemjWBqOosVR1Vze1D4KSIRAE4fzOqeYoUwPsTE+OsmwLEichRYC0wRERWNee+mIb59dXDiQntzL2vxHMkq9DtcEwbo6o8/P4uNhzO5o83jqZnd+s8b03cbMJaDJw9q2o+8GE1ZZYBl4tIqNN5fjmwTFX/V1V7q2o/YBpwQFVntEDMpp5CggJ44a6JCPCdF7eQX1zmdkimDVmw+jBvb03mxzMHc/34GLfDMVW4mUCeBGaLSBIwy1lGROJE5DkAVc3B09exxbk97qwzbUjfHl34920XcCSrkPe3pbgdjmkjissq+MfnB5k1PJKfzRrsdjimGq5dSKiq2cDMatbHA/d4LS8EFp7neY4CdrlzKzdlYA+G9OzKxwlpzJ/az+1wTBuw+kAmBSXl3DGlHyLWcd4a2WCKpsVcPaY3W47lkJ5nE0+Z2i1JSCM0yJ+pA3u4HYqpgSUQ02KuGhOFKnyyK632wqZDKyqt4LO9J5k7Kgo/X/uaaq3snTEtZmBEV4ZHdWeJDW9iarFqfwZnSiu4ZkyU26GY87AEYlrU1WOi2HY8l5TcIrdDMa3YkoQ0wrsGMKl/mNuhmPOwBGJa1NXOL8pPrRnL1OBMaTkr9p3kCmu+avXs3TEtqm+PLozs3Z2lieluh2JaqdUHsiguq+TK0dZ81dpZAjEtbuawSLYdP0XumVK3QzGt0Mp9GXQL9COuX7UzPJhWxBKIaXEzhkVSqbA6KcvtUEwro6qs3J/BJYMj8Lfmq1bP3iHT4sbGhBDWJYBV+6ob/sx0ZHvS8sk4XcKlwyLdDsXUgSUQ0+J8fYTpQyJYdSCTiso6D8JsOoCVzo+K6UNs5Oy2wBKIccWMoRHkFJaSkJzrdiimFVm5P5MxMcFEdOvkdiimDiyBGFdMHxKBj3i+MIwBOFVYyvbjp5gx1Jqv2gpLIMYVIUEBTIgNPddkYczqpEwqFS6z/o82wxKIcc2lwyLZlZLH1mM2Qn9HV1GpvLrxGOFdAxgTHex2OKaOLIEY19wxpS99wjrz0zd3cNommurQnv3iEFuOnuKhK4bjY3OetxmWQIxrugX688y3xpFyqohHF+92Oxzjkp0ncnl6+QGuHhPFNyZEux2OqQdLIMZVF/QN40eXDea9bSl8tuek2+GYFlZZqfzsrR1EduvE764bbRNHtTGWQIzrfnTZILoH+vH5futQ72iOZhdyOLOQH80cTHCQv9vhmHqqcwIRkaCmelERCROR5SKS5PytdtAbEZnvlEkSkfle6wNEZIGIHBCRfSJyQ1PFZlqen68Po6KD2Z2S53YopoUlpuYDntEJTNtTawIRkakisgfY5yyPFZF/N/J1HwRWqOpgYIWzXPV1w4BHgcnAJOBRr0TzMJChqkOAEcAXjYBN6I0AAB+3SURBVIzHuGxUdDB7009TVlHpdiimBe1OySPA14fBPbu6HYppgLrUQJ4G5gDZAKq6E7ikka87D3jJuf8ScF01ZeYAy1U1R1VPAcuBuc5j3wH+4MRTqao2Kl8bN7J3d0rLKzmYUeB2KKYFJabmMSyqmw2c2EbV6V1T1RNVVlU08nV7qurZGYXSgZ7VlIkGvF83GYgWkbN13d+KyDYReVtEqtseABG5V0TiRSQ+M9Ouem6tRjnn/u+yZqwOQ1VJTMlnZG+77qOtqksCOSEiUwEVEX8ReQDYW9tGIvKZiCRWc5vnXU5VFajPiHp+QAywXlUnABuAP9dUWFUXqGqcqsZFRNgAba1V/x5d6BLga/0gHUjyqSLyisoYFd3d7VBMA/nVocx9wN/w1AhSgP8AP6htI1WdVdNjInJSRKJUNU1EooDqTr9JAWZ4LccAq/A0pZ0B3nPWvw3cXetemFbNx0cY0bv7uU5V0/7tTvX8WBhlNZA2q9YaiKpmqeptqtpTVSNV9duqmt3I110MnD2raj7wYTVllgGXi0io03l+ObDMqbF8xJfJZSawp5HxmFZgZO9g9qTm2xDvHURiSj6+PsLQXt3cDsU0UK01EBF5gWqamFT1O4143SeBt0TkbuAYcJPzWnHAfap6j6rmiMhvgS3ONo+r6tlBk/4beEVEngEygbsaEYtpJUZFB/Pi+qMcySpgUKR9qbR3ial5DI7sSqC/r9uhmAaqSxPWEq/7gcD1QGpjXtSpwcysZn08cI/X8kJgYTXljtH4M8FMK3O2LTwxJZ/YsC5sOZrD1IE97OrkduRQZgF+PkJsWBCJKXk2dHsbV2sCUdV3vZdFZBGwttkiMh3WoIiudPLzYfmek7yw7gg7k/N47o44Zo2o8SQ708bc/eIWUvOK+dGlg8gqKGVUb+tAb8sacvL1YMB+Npgm5+frw7Co7ny8K43DWYUE+Pmw/lBju9tMa5GaW8TR7DN0D/TnL8sPAF+evm3aprr0gZzG0wcizt90PH0QxjS5GyZEE9LZnyeuG8Uv30lg0xFLIO3F2ffyxbsm8sWBTNYkZVoCaePq0oRlvZmmxdwxpR93TOkHwKT+Yfz98yTyisoI7mwD7bV1mw7n0C3Qj+FR3RkVHcwPLh3kdkimkWpMICIy4Xwbquq2pg/HmC9NHhCGroD4oznMHG79IG3dpiM5TOoXhq9NGNVunK8G8pfzPKbAZU0cizFfMSE2lABfHzYfsQTS1mXkF3Mkq5BbJvVxOxTThGpMIKp6aUsGYkxVgf6+jO0TzMYjNmd6W7fJeQ8n9+/hciSmKdXlOhBEZBSeYdMDz65T1ZebKyhjzprUP4xnvzhMQUk5XTvV6eNqWqFNR7LpEuDLSDttt12py3wgjwL/cG6XAk8B1zZzXMYAnl+sFZXK1mOn3A7FNMKmwzlc0C8MPxu2vV2py7t5I56rxtNV9S5gLGDn3pkWcUHfUHx9hI2H7XTetiqroISkjAIm9w9zOxTTxOqSQIpVtRIoF5HueEbOtZ4w0yK6dPJj2qBwXtt4jMzTJW6HYxrgHyuSEIGZw+364/amxgQiIv8SkWnAZmcSp/8DtgLb8MzBYUyLeOSaERSXVfL4Eht0ua3ZdvwUL288xvwp/RjWy/o/2pvz9UoeAP4E9AYKgUXAbKC7qia0QGzGADAwois/vGwQf11+gG+Mj+bSYfZLti0oq6jkoXd30at7IA/MGep2OKYZ1FgDUdW/qeoUPKPeZuMZFXcpcL2IDG6h+IwB4L7pAxkc2ZVffZBIVoE1ZbUFf/ssif0nT/PbeaPsDLp2qi4TSh1T1T+q6njgFuA6YF+zR2aMlwA/H/78zbFkF5Ywf+Fm8ovL3A7JnMfLG47yz5UHuSkuxkZTbsfqchqvn4hcIyKvAZ8C+4FvNHtkxlQxtk8Iz377Ag6cPM09L8ZTVFrhdkimGh9sT+GRD3cza3hPfn/9aLfDMc3ofJ3os0VkIZAMfBf4GBioqjeranVT0BrT7GYMjeTpb41jy7Ec/rkyye1wTBUn84v55TsJXDggjH/eOt6u+2jnzvfuPgSsB4ar6rWq+rqqFrZQXMbU6OoxvZk7shevbjxOYUm52+EYLy+uP0p5ZSVP3TDWpqrtAM7XiX6Zqj6nqnYJsGl17rl4AHlFZbwdf8LtUIyjoKSc1zYeY+6oXsT2CHI7HNMCXKtfikiYiCwXkSTnb2gN5eY7ZZJEZL7X+ltEZJeIJIjIUhEJb7nojdsu6BvKBX1DeX7dESoq1e1wDPDWlhPkF5fz3YsHuB2KaSFuNlA+CKxQ1cHACmf5K0QkDHgUmAxMAh4VkVAR8QP+BlyqqmOABOCHLRa5aRW+e3F/TuQUsWx3utuhdHjlFZUsXHeEuL6hjI+t9regaYfcTCDzgJec+y/hOT24qjnAclXNcZrSlgNz8UyvK0AXERGgO5Da/CGb1mT2iF707RHEgtWHUbVaiJuW7k4n+VQR373Eah8diZsJpKeqpjn304HqThaPBrwbuZOBaFUtA+4HduFJHCOA56t7ERG5V0TiRSQ+MzOzyYI37vP1Ee6e1p8dJ3JttF4XqSr/t/ow/XoEMcsm/upQmjWBiMhnIpJYzW2edzn1/Hys809IEfHHk0DG4xlqJQHPWWNfo6oLVDVOVeMiIiIavjOmVbrxghhCgvxZsPqw26F0WFuOnmJnch53XzzApqvtYJp1fAFVnVXTYyJyUkSiVDVNRKLwjPJbVQoww2s5BlgFjHOe/5DzXG9RTR+Kaf+CAvy4/cK+/HPlQY5kFdI/vIvbIXU4/7fmMKFB/tw4IcbtUEwLc7MJazFw9qyq+UB1FycuAy53Os5DgcuddSnACBE5W6WYDext5nhNK3X7lL74+/jw/FqrhbS0Q5kFfLb3JLdf2JfOAXbdR0fjZgJ5EpgtIknALGcZEYkTkecAVDUH+C2wxbk97nSopwK/AVaLSAKeGsnvXdgH0wpEdgvk+vHRvB2fTE5hqdvhdCgL1x7B39eH26f0czsU4wLXhshU1Ww8Mx1WXR8P3OO1vBDPSMBVyz0LPNucMZq24+ZJfXgz/gQbDmVz1Zgot8PpMD7fl8HsET2J6NbJ7VCMC2ygGtMujOjdnQBfHxJSct0OpcPIOF1MWl4x4/uEuB2KcYklENMudPLzZVhUN3Yl57kdSoeRmOI51mNiLIF0VJZATLsxKjqYXSl5dlFhC9mVnI8IjOxtU9V2VJZATLsxJjqY08XlHMs+43YoHcKulFwGRnSli8022GFZAjHtxqjoYAASUqwZqyUkJOcx2jnmpmOyBGLajSE9uxHg53Oubd40n5P5xWScLrEE0sFZAjHtRoCfD8OjupOQbGdiNbezJyuMibEE0pFZAjHtyujo7iSm5FNpc4Q0q4SUPHzEc/q06bgsgZh2ZUx0CAUl5RzNttmXm1NiSh6DIrsSFGAd6B2ZJRDTrox2mlR2WT9Is1FVpwPdrv/o6CyBmHZlcGRXOvn5sP249YM0l9S8YrIKShgdbc1XHZ0lENOu+Pn6cPHgCD7YkUJBSbnb4bRLr248hghcMsTm1+noLIGYdueHlw0i90wZr2485nYo7U7umVJeXn+Uq0ZHMSCiq9vhGJdZAjHtzrg+IVw8OJzn1hymqLTC7XDalYXrjlJYWsEPLxvkdiimFbAEYtqlH88cTFZBKa9vPu52KO1GfnEZL6w7wpyRPRnWy/o/jCUQ005N7BfGhQPC+H9fHKK8otLtcNqF1zYe53RxOT+6bLDboZhWwhKIabfmT+lHxukSttkZWU3i08Q0JsSGnBtzzBhLIKbdmjY4HD8fYeX+DLdDafMyT5eQkJzHzOE93Q7FtCKuJBARCROR5SKS5PwNraHcUhHJFZElVdb3F5FNInJQRN4UkYCWidy0Jd0C/ZnYL4yV+yyBNNYqJwnPGGqn7povuVUDeRBYoaqDgRXOcnX+BNxezfo/Ak+r6iDgFHB3s0Rp2rxLh0WwL/00qblFbofSpq3an0lkt06MiLLOc/MltxLIPOAl5/5LwHXVFVLVFcBp73UiIsBlwDu1bW/MpUMjAc8XIMCTn+7jwXcT3AypTYg/msO1/1xLSm4RZRWVrE7K5NKhkXj+/YzxcCuB9FTVNOd+OlCfhtUeQK6qnr3MOBmIrqmwiNwrIvEiEp+ZmdmwaE2bNSiyKzGhnVm5P4NPdqXx7BeHeDP+BFkFJW6H1qq9vvk4Ccl5/PSN7Ww5ksPp4nIuHRbpdlimlWm2BCIin4lIYjW3ed7l1DOBdbONva2qC1Q1TlXjIiKs/bajEREuHRrJ2qQsHnw3gT5hnVGFz/acdDu0VqusopIVezPoE9aZLUdP8Yu3d+LvK1w0qIfboZlWptkSiKrOUtVR1dw+BE6KSBSA87c+vZzZQIiInB1HOgZIadroTXty6bAIisoqqFR49e7J9AnrzNLd6W6H1WptOpxDXlEZv7pqBNePjyYtr5iJ/cLoFujvdmimlXGrCWsxMN+5Px/4sK4bOjWWlcCNDdnedDxTBoQzITaEp24cQ98eXZgzohfrD2aTX1zmdmit0rLd6QT6+3DJ4AgenzeSCweEccukWLfDMq2QWwnkSWC2iCQBs5xlRCRORJ47W0hE1gBvAzNFJFlE5jgP/TfwcxE5iKdP5PkWjd60KZ0DfHnv+xdx5egoAOaO6kVpRaWd3luNykpl2e50ZgyJpHOAL90C/Xnj3ilcM7a326GZVsiV6cRUNRuYWc36eOAer+WLa9j+MDCp2QI07dqE2FDCu3biP7tPMm9cjedfdEg7knPJOF3CnFF2waCpnV2JbjocHx/h8pE9Wbk/g+IyG63X27LEdPx8hMuGWQIxtbMEYjqkOSN7caa0grVJWW6H0mqoepqvpgzsQXBn6zA3tbMEYjqkKQN60C3Qj2V2NtY5B04WcDT7DHNH9XI7FNNGWAIxHVKAnw8zh0Xy2d6TNty7Y2liOiIwe4Q1X5m6sQRiOqy5o3px6kwZm4/muB1Kq7BsdzoXxIYS2S3Q7VBMG2EJxHRYlwyJoJOfD8sSrRnrRM4Z9qTlM2ekNV+ZurMEYjqsoAA/pg+JYNnuk1RWNttoOm3C2b4gSyCmPiyBmA5tzshepOcXk5CS53YorlqamM7wqO7E9ghyOxTThlgCMR3azOGR+PkInyam1V64ncrIL2br8VPMtdqHqSdLIKZDCwkKYMbQSN7dmkxJece8qHDR5hOowrXjbLgSUz+WQEyHN39qX7IKSvlkV8erhZRVVPLapmNMHxJB//Aubodj2hhLIKbDmzYonAERXXhx/TG3Q2lxSxPTyThdwp1T+7kdimmDLIGYDk9EmD+lHztP5LLjRK7b4bSolzccJTYsiOlDbLI1U3+WQIwBvjEhmi4Bvry8/qjbobSY3al5bDl6ijum9MXHx+Y6N/VnCcQYoFugPzdeEMNHCansSc13O5xmp6r8edl+Ovv78s0L+rgdjmmjLIEY4/jxzMGEBgXwo0XbOFNa7nY4zerF9UdZuT+TX84dSnCQjbxrGsYSiDGOHl078fS3xnE4q5DHP9rjdjjNJjEljz98so9ZwyOt89w0iiszEhrTWl00KJz7pw/k36sOkXyqiLh+ocwbF93mT3Etr6jk/e0pbDqSw6r9GYR28eepG8ciYn0fpuEsgRhTxc9mD6FClS/2Z/K3FUm8vuk4a//7MgL82m6FfcGawzy1dD9hXQKYEBvKT2YOJqxLgNthmTbOlf8IEQkTkeUikuT8Da2h3FIRyRWRJVXWvyYi+0UkUUQWiog14pom4+/rw0NXDGfpTy9h4fyJZJwuadMXGZZVVPLy+mNMGxTO1l/N4rn5cYyOCXY7LNMOuPWT6kFghaoOBlY4y9X5E3B7NetfA4YBo4HOwD3NEaQx04dEMDCiCwvXHUG1bY7Y+2liOun5xdw9rb81WZkm5VYCmQe85Nx/CbiuukKqugI4Xc36T9QBbAZimitQ07H5+Ah3XtSfhOQ8th475XY4DbJw7REGhHexiwVNk3MrgfRU1bNtAulAg+bQdJqubgeWnqfMvSISLyLxmZmZDXkZ08HdMCGa7oF+LFx3xO1Q6m3b8VPsOJHLnRf1s4sFTZNrtgQiIp85fRRVb/O8yzm1iIa2DfwbWK2qa2oqoKoLVDVOVeMiIuwXmKm/oAA/bpkcy9LEdJJPnXE7nHpZuPYI3QL9uGGCVdJN02u2BKKqs1R1VDW3D4GTIhIF4PzNqO/zi8ijQATw86aN3Jivu2NKP0SEVza0nQEXU3OL+DQxnVsmxdKlk51waZqeW01Yi4H5zv35wIf12VhE7gHmALeoamUTx2bM10SHdGbuqF4s2nycwpK2cZX6yxuOoarcMaWv26GYdsqtBPIkMFtEkoBZzjIiEiciz50tJCJrgLeBmSKSLCJznIeexdNvskFEdojIIy0bvumIvnNRf/KLy3lvW7LbodTqTGk5izYfZ+6oXsSE2jS1pnm4Uq9V1WxgZjXr4/E6JVdVL65he6uPmxY3ITaEsX1CeGHdUW6b3LpHsH1/ewp5RWV856L+bodi2rG2e2mtMS1MRPjORf04nFXI5/vq3W3XYsoqKnlh3VFGRwdzQd9qr9E1pklYAjGmHq4YFUVMaGfuf20rf/hkL6eLy9wO6StW7s9g7jOrOZhRwPemD7ALB02zsgRiTD0E+Pnw3vencv34aBasOcwVf1tDUWmF22EB8OwXh7jrhS1UKjw/P46rx/R2OyTTzlkCMaaeIrsF8tSNY1l450SSTxXxVvwJt0OisKSc/111iOlDIlj200uYObxB1+YaUy+WQIxpoEuHRnJB31D+b81hyisqUVUeW7ybxxbvbvbXTkzJ45p/rOVIViEAizYfJ6+ojJ/MGtymRw02bYt90oxphPumDyT5VBEf70rjlY3HeHH9Ud7YcpyS8uZt1vpoZyq7UvK475Wt5BWV8fzaI0zqH8aEWOs0Ny3HEogxjTBzWCSDI7vy1NL9/HbJHqJDOlNcVsmO47nnyvxr5UHWH8xq8GuUlFfwyIeJnMj5chiV9YeyiQoO5EDGaa771zrS8oq5f/rARu2LMfVlCcSYRvDxEe69ZAApuUX0DunMG/deiI/AukPZAKTkFvGnZft5bm3DB2LcevQUL284xgvrjgKQd6aMxNQ8bp4YywOXD+VIViHDenVjxlAb6820LLsgz5hGmjcumhM5Z7h2XG/6hAUxOjqYDYeyYPYQPknwDDq9/fgpVLVBp9VuO+4ZRv6TXWn86qrhbDicjSpMHdSDC5wmq4sGhdspu6bFWQ3EmEYK8PPh55cPZVBkNwCmDgpn+/FcCkvKWZKQCsCpM2UczW7YSL7bneaw9Pxi4o+dYsOhLIICfBkbE4KPj/CDSwcxrk9I0+yMMfVgCcSYJjZ1YA/KK5X3tiWzMzmPb0yIBjy1kPpSVbafyOXK0b3o5OfDkoRU1h/KZmK/MDvbyrjOPoHGNLG4vmH4+wp//s8BAH46cwhdO/mda4oqLCnnp29sZ+uxnK9tW1peyUPv7WLF3pMAHMs+Q05hKdMGRXDZsEg+2J5CUkYBUwf2aLkdMqYGlkCMaWKdA3wZHxtKXlEZ4/qEENsjiHF9Qs41RX2wI4UPdqRy78tbSc0t+sq2T3y8h0Wbj/O7T/Y6tQ9P0pnQN4Srx/Qmv9gzlPzUgeEtu1PGVMMSiDHN4GwN4eoxUQCMjw1hX/ppzpSW8/qm48SGBVFSXsl9r26luMxzzchb8Sd4ecMxRvbuzuHMQjYezmHbsVy6BPgyOLIblw2LJCjAl+6Bfozo3d21fTPmLDsLy5hmcO3Y3mw4lM28cZ7+j/GxIVRUKq9uPMbu1Hx+O28kvYI7892X45nzzGqCO/uzL+00Fw3qwf+7PY6pf1jB65uPcySrgLF9QvD1EToH+HLf9IFUquLbioeSNx2HJRBjmsGAiK68+b0p55bH9/Gcbvv08iQ6+/syb3w03QP9+eMNo1mamA7A1WOj+NVVI+jayY9vTIjhtU3HqFS+coHgj2cObtkdMeY8LIEY0wJCuwTQP7wLR7IK+VZcH7oH+gPwrYmxfGti7NfK3zY5lhfXHwU8tRdjWiPrAzGmhZxNBLdM/nrCqGpwz25M7BfqbGfjW5nWyZUaiIiEAW8C/YCjwE2q+rWT5EVkKXAhsFZVr67m8b8D31HVrs0asDFN4K6p/enfowtjY4LrVP5/rvRcdR7WJaCZIzOmYdyqgTwIrFDVwcAKZ7k6fwJur+4BEYkD7KeZaTNGxwTzo5mD6zzkyPjYUL4/Y1AzR2VMw7mVQOYBLzn3XwKuq66Qqq4ATlddLyK+eJLLL5srQGOMMefnVgLpqappzv10oL7Tp/0QWOz1HMYYY1pYs/WBiMhnQK9qHnrYe0FVVUS0Hs/bG/gmMKOO5e8F7gWIja2989IYY0zdNFsCUdVZNT0mIidFJEpV00QkCsiox1OPBwYBB5225CAROaiq1TYWq+oCYAFAXFxcnROVMcaY83OrCWsxMN+5Px/4sK4bqurHqtpLVfupaj/gTE3JwxhjTPNxK4E8CcwWkSRglrOMiMSJyHNnC4nIGuBtYKaIJIvIHFeiNcYY8zWuXAeiqtnAzGrWxwP3eC1fXIfnsmtAjDHGBXYlujHGmAYR1Y7TrywimcCxBm4eDmQ1YTgtzeJ3l8XvLou/cfqqakTVlR0qgTSGiMSrapzbcTSUxe8ui99dFn/zsCYsY4wxDWIJxBhjTINYAqm7BW4H0EgWv7ssfndZ/M3A+kCMMcY0iNVAjDHGNIglEGOMMQ1iCaQORGSuiOwXkYMiUtPkV62CiPQRkZUiskdEdovIT5z1YSKyXESSnL+tejIuEfEVke0issRZ7i8im5z34E0RabXT9IlIiIi8IyL7RGSviExpS8dfRH7mfHYSRWSRiAS25uMvIgtFJENEEr3WVXu8xePvzn4kiMgE9yI/F2t18f/J+fwkiMj7IhLi9dhDTvz73R7eyRJILZzJq/4FXAGMAG4RkRHuRnVe5cAvVHUEnumAf+DEW9dZIFuLnwB7vZb/CDztDJx5Crjblajq5m/AUlUdBozFsx9t4viLSDTwYyBOVUcBvsDNtO7j/yIwt8q6mo73FcBg53Yv8L8tFOP5vMjX418OjFLVMcAB4CEA53/5ZmCks82/ne8oV1gCqd0k4KCqHlbVUuANPDMqtkqqmqaq25z7p/F8eUVTx1kgWwMRiQGuAp5zlgW4DHjHKdJq4xeRYOAS4HkAVS1V1Vza0PHHM0ZeZxHxA4KANFrx8VfV1UBOldU1He95wMvqsREIcaaUcE118avqf1S13FncCMQ49+cBb6hqiaoeAQ7i+Y5yhSWQ2kUDJ7yWk511rZ6I9MMzf8omGj8LZEt6Bs90xZXOcg8g1+sfqjW/B/2BTOAFpwnuORHpQhs5/qqaAvwZOI4nceQBW2k7x/+smo53W/x//g7wqXO/VcVvCaSdEpGuwLvAT1U13/sx9Zy73SrP3xaRq4EMVd3qdiwN5AdMAP5XVccDhVRprmrlxz8Uz6/c/kBvoAtfb15pU1rz8a6NiDyMp1n6NbdjqY4lkNqlAH28lmOcda2WiPjjSR6vqep7zuqTZ6vqDZgFsiVdBFwrIkfxNBdehqdPIcRpUoHW/R4kA8mquslZfgdPQmkrx38WcERVM1W1DHgPz3vSVo7/WTUd7zbz/ywidwJXA7fplxfstar4LYHUbgsw2DkLJQBPB9Zil2OqkdNf8DywV1X/6vVQg2eBbEmq+pCqxjizTd4MfK6qtwErgRudYq05/nTghIgMdVbNBPbQRo4/nqarC0UkyPksnY2/TRx/LzUd78XAHc7ZWBcCeV5NXa2GiMzF04x7raqe8XpoMXCziHQSkf54TgbY7EaMAKiq3Wq5AVfiORPiEPCw2/HUEus0PNX1BGCHc7sSTz/CCiAJ+AwIczvWOuzLDGCJc38Ann+Ug3hmqezkdnzniXscEO+8Bx8AoW3p+AO/AfYBicArQKfWfPyBRXj6a8rw1ADvrul4A4LnrMpDwC48Z5u1xvgP4unrOPs//KxX+Yed+PcDV7gZuw1lYowxpkGsCcsYY0yDWAIxxhjTIJZAjDHGNIglEGOMMQ1iCcQYY0yDWAIxppFE5GFn9NoEEdkhIpOb8bVWiUhccz2/MfXhV3sRY0xNRGQKnquFJ6hqiYiEA61mqHNjmpPVQIxpnCggS1VLAFQ1S1VTReQREdnizKmxwLmq+2wN4mkRiXfmCpkoIu8581Y84ZTp58wF8ZpT5h0RCar6wiJyuYhsEJFtIvK2M/6ZMS3GEogxjfMfoI+IHBCRf4vIdGf9P1V1onrm1OiMp5ZyVqmqxgHP4hli4wfAKOBOEenhlBkK/FtVhwP5wPe9X9Sp6fwKmKWqE/Bc+f7z5tlFY6pnCcSYRlDVAuACPJMTZQJvOoPgXerM4LcLz4CQI702OzuW2i5gt3rmcCkBDvPlQHknVHWdc/9VPEPUeLsQzwRn60RkB57xnvo26c4ZUwvrAzGmkVS1AlgFrHISxveAMXjGWTohIo8BgV6blDh/K73un10++z9ZdYyhqssCLFfVWxq9A8Y0kNVAjGkEERkqIoO9Vo3DM8gdQJbTL3Hj17esVazTQQ9wK7C2yuMbgYtEZJATRxcRGdKA1zGmwawGYkzjdAX+ISIheCb+OYinOSsXz2i26XimBKiv/Xjms1+IZzj1r8zdraqZTlPZIhHp5Kz+FZ5Ro41pETYarzGtjDMV8RKnA96YVsuasIwxxjSI1UCMMcY0iNVAjDHGNIglEGOMMQ1iCcQYY0yDWAIxxhjTIJZAjDHGNMj/B22bEVUzrx2PAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEWCAYAAACNJFuYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADt0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjByYzMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy9h23ruAAAgAElEQVR4nOy9d5wcd33//3pvL7d7e10n3alasmS5YFs2NjYY3DAYMNWhBAgJIbSQQvmFEAIJIZhOiGkGvgEDMRhjSmyDK7Zxt2TZlmVJVtcVXW/b6+f3x2c+s5+Zndlyt3t70n2ej8c97m52duazszOfd39/iDEGhUKhUChkHM0egEKhUCiWHko4KBQKhaIEJRwUCoVCUYISDgqFQqEoQQkHhUKhUJSghINCoVAoSlDCQWELEd1PRO8t8/p3iejTizmm5Q4R/Z6I3l3H45X9jhXLFyUcThCI6AgRJYkoRkQjRPQjImpZxPP/BRE9JG9jjL2fMfa5BpxrLREx7bPGtM/+T9LrREQfJ6L92jU5RkT/SUQeaZ8face4xnTsr2vb/0L6XHnpXDEiut5mXPdr7z3LtP3X2vaX1/M6WMEYexVj7MfS2B+q9J5GUOk7Upz4KOFwYvFaxlgLgBcBOBvAJ5s8nkYT0T7v2wD8KxFdpW3/JoD3AXgXgBCAVwG4HMDPTe9/QdsHAEBELgDXAjho2u9RxliL9PPhMmMyH7MDwIUAxmv9cCcJ4jt6M4BPE9EVzR6Qoj4o4XACwhgbAXAnuJAAABDRBUT0CBHNENEzsharaZiHiChKRIeJ6B3a9s8S0U+l/YQ26JLPR0RbAHwXwIWaljijbf8REf2H9vfLiWiQiD5KRGNEdJyI3iMdo4OI/o+I5ojoSSL6j2q1XsbYowB2AzidiDYC+CCAdzDGHmWM5RhjuwG8CcDVRHSJ9Nb/A3AxEbVp/18F4FkAI9Wc14afAfgzInJq/78NwK8BZKTPej4RPap9F8eJ6HqTVXMlEe0jolki+jYRPSBcO8IaIKKvENG09n29Snrv/UT03jLficFNZLYuiOgKItqrnft6ACR/OCL6SyLao537TiJaU81FYYxtB/+O5HvS8lia5fd17T6ZI6JdRHS69tqPiLsr79bu1wfkMRDRS7T7Z1b7/RLTtfkcET2svfcuIurUXvMR0U+JaFL7Xp4koh7ttVYi+qH2XQ1p96b4fpctSjicgBBRH7i2fED7fxWA2wH8B4B2AB8D8Csi6iKiILim/SrGWAjASwA8Xcv5GGN7ALwfRQ07YrPrCgCtAFYB+CsA35Im5m8BiGv7vFv7qeazEhFdBGArgJ0ALgMwyBh7wjTGAQCPAbhS2pwC8FsAb9X+fxeAG6s5bxmGATwvncfqmHkA/wCgE9yquAxcoEGbrG4Bt/o6AOwD/05kXqxt7wTwJQA/JCLDJF7Dd6KjnftWAP+iHfsggIuk168B8M8A3gigC8CfANxU6bjaey8AcDqK92S5Y10J4GUANoHfL9cCmJQO9w4An9PG+DS4QAYRtYPf598Ev3ZfA3A7cetN8HYA7wHQDcAD/iwA/H5rBdCvvff9AJLaaz8CkANwCrhFfiWAZR+HUcLhxOI3RBQFMABgDMBntO1/DuAOxtgdjLECY+xuANsBvFp7vQCudfsZY8c1TbsRZAH8O2Msyxi7A0AMwKmaFvYmAJ9hjCUYY88D+HEVx5sAMAXgBwD+iTF2L/iEcdxm/+PgE5HMjQDeRUQRAJcA+I3F+y7QtEnxc0GFcYljbgZ3qzwqv8gY28EYe0yzao4A+J52boB/J7sZY7cyxnLgE53ZkjnKGPs+YywPfp16AfRUGFM1iHPfwhjLAviG6dzvB/AFxtgebWz/CeBFFayHCSJKAngUwLdRvL7ljpUFdwduBkDaPvJ3ejtj7EHGWBrAp8Cto34AVwPYzxj7iXZtbwKwF8Brpff+D2PsBcZYEsDNKFoyWXChcApjLK99R3Oa9fBqAH/PGIszxsYAfB1FhWLZooTDicXrNe3/5eAPVqe2fQ2At8gTHICLAfQyxuIA/gz8YT1ORLdrk1ojmNQmAkECQAv4hO0CF2oC+W87OhljbYyxLYyxb2rbJsAnSyt6tdd1GGMPaef/FIDbtEnDzGOMsYj081iFcd0K4FIAHwbwE/OLRLSJiG4jnjgwBz4xiu9qJaTPznjny0HTIUak1xPan/VIPrA6t/w9rAHwX9I9NAXudlpV5pid2tg+Cn5fuisdizF2H4Drwa3JMSK6gYjC0jHlMca0967Ufo6azn/UND5Z2In7D+Df050Afk5Ew0T0JSJya+N0gz8bYqzfA7c8ljVKOJyAMMYeADeFv6JtGgDwE9MEF2SMXaftfydj7ArwyXMvgO9r74sDCEiHXlHutAsY8ji42d4nbeuf57HuA9BPROfLGzXN8gIA91u856fgk9dCXUoA9An79wA+AAvhAOA74Nd5I2MsDO5eEW6h45Cug+Yu6is5QpVDsdhW7js9Dum6a+eWv4cBAH9juo/8jLFHyg6Ca+JfA3fjfbCaYzHGvskYOxfAaeDupY9Lh5TH2ALuKh3WfsxWzGoAQ+XGp50vyxj7N8bYaeBuvNeAuwQHAKTBFRExzjBjbGulY57sKOFw4vINAFcQT6v8KYDXEtEricipBd9eTkR9RNRDRNdosYc0uKunoB3jaQAvI6LVRNSK8tlPowD6SAqsVovmHrkVwGeJKKBZLu+q8Da7Y70AHoj9GfEgvJOItgL4FYBHANxj8bZvArgCwIPzOacN/wzgEs1tZCYEYA5ATPusH5Beux3AGUT0euKB/w+hvFAuh9V38jSAN2rX+RTw2I987q1E9Ebt3B8xnfu7AD6pXU8RqH1LDeO5DsAniMhX7lhEdB4RvVjT3OPgQqUgHefVRHSx9rk+B27ZDQC4A8AmIno7EbmI6M/AhcttlQZGRK8gojM0F+ccuJupoLmz7gLwVSIKE5GDiDaQMbFhWaKEwwkKY2wcXBP+V+3BEQHAcXBt6OPg368DwD+Ca11T4L7vD2jHuBvAL8AzeHag/EN2H3g2yggRTZTZz44PgwcER8C17ZvAhdV8+DB4HOKn4K6D58DdC69njBXMOzPGphhj92pulLrAGBvWXFZWfAw8MBoFt9J+Ib1vAsBbwAPNk+CT23bM71pYfSdfB8+cGgWPV/zM4tzXaefeCOBh6fVfA/giuOtlDvy66plSVXA7gGkAf13hWGHw6zIN/r1NAviydJz/BY+nTQE4FzymBsbYJLjG/1HtPZ8A8Brtc1ViBXgiwByAPQAeQNHqexd48Pp5bUy3wN51uWygOj4vCkXVENEXAaxgjC242peI/g3AGwC8jDE2s+DBLSJE5ACPObyDMfbHZo+n2RDRj8Cz0f6l2WNZ7ijLQbEoENFmIjpTS009H9zd8et6HJsx9hkAN4DHHJY8mvsvQkReFOMRlYLgCsWi4qq8i0JRF0LgrqSV4C6Pr4LXINQFxphly4slyoXgrhPhyni9TRaVQtE0lFtJoVAoFCUot5JCoVAoSmiaW0nLS78RvPKTAbiBMfZfWon8LwCsBXAEwLWMselyx+rs7GRr165t6HgVCoXiZGPHjh0TjDFzVwEATXQrEVEveAXvU0QUAk+lfD2AvwAwxRi7jngL4DbG2P9X7ljbtm1j27dvb/iYFQqF4mSCiHYwxrZZvdY0t5LW4+cp7e8oeO7xKvB8fdF358fgAkOhUCgUi8iSiDkQ0VrwboiPA+iRmnCNwKbhGBG9j4i2E9H28fHl2kpfoVAoGkPThYPWO+VX4F0R5+TXtIpWS78XY+wGxtg2xti2ri5Ll5lCoVAo5klThYPWW+VXAH7GGLtV2zyqxSNEXGKsWeNTKBSK5UrThIPWEfKHAPZoHR0Fv0NxIZh3o46FUgqFQqGojmZWSF8E4J0AdhGRWJnsn8Gbgt1MRH8F3pTr2iaNT6FQKJYtTRMOWkdLsnn5ssUci0KhUCiMND0g3UwOjMXwtbtfwEP7J6DaiCgUCkWRZd14b8/xOVx/334UGHD1mb34+rUvgse1rOWlQqFQAFjmwuG1Z63EFaf14IcPHcaX79yHVr8b//mGM5o9LIVCoWg6y1o4AIDP7cSHXnEK5lJZfO+BQ7j6jF5cdEpn5TcqFArFSYzyoWj8w+Wb0Nfmx5f+sFfFHxQKxbJHCQcNn9uJv3nZejwzOIudAyfUSpMKhUJRd5RwkHjjOX0I+Vz40cNHmj0UhUKhaCpKOEgEvS68+dw+/P6544imss0ejkKhUDQNJRxMvPqMXmTzDA+8oDq9KhSK5YsSDibOWd2GjqAHd+0ebfZQFAqFomko4WDC6SBctqUbf9w7hkyu0OzhKBQKRVNQwsGCSzf3IJrOYdeQylpSKBTLEyUcLNi2tg0A8OSR6SaPRKFQKJqDEg4WdLZ4sb4ziO1Hppo9FIVCoWgKSjjYsG1tG3YcnUahoKqlFQrF8kMJBxu2rWnHdCKLQxOxZg9FoVAoFh0lHGw4Z00EAPD0wGyTR6JQKBSLjxIONqztCMLjcmDfyFyzh6JQKBSLjhIONricDmzsbsHekWizh6JQKBSLjhIOZTh1RQj7lHBQKBTLECUcyrB5RQhj0TSm45lmD0WhUCgWFSUcynDqijAAKNeSQqFYdijhUIbNK0IAoILSCoVi2aGEQxm6Q16EvC4cnog3eygKhUKxqCjhUAYiQn97AMemEs0eikKhUCwqSjhUYLUSDgqFYhmihEMF+tv9GJxOqh5LCoViWaGEQwVWtweQzhUwHks3eygKhUKxaCxZ4UBEVxHRPiI6QET/1Kxx9LcHAAADyrWkUCiWEUtSOBCRE8C3ALwKwGkA3kZEpzVjLEI4qLiDQqFYTixJ4QDgfAAHGGOHGGMZAD8HcE0zBrIq4geREg4KhWJ5sVSFwyoAA9L/g9o2HSJ6HxFtJ6Lt4+PjDRuIz+3EirBPCQeFQrGsWKrCoSKMsRsYY9sYY9u6uroaeq5VET+Oz6Qaeg6FQqFYSixV4TAEoF/6v0/b1hR6Wn0YnVPCQaFQLB+WqnB4EsBGIlpHRB4AbwXwu2YNpiekhINCoVheLEnhwBjLAfgwgDsB7AFwM2Nsd7PGs6LVi3gmj2gq26whKBQKxaLiavYA7GCM3QHgjmaPAwB6wj4AwOhcCiGfu8mjUSgUisazJC2HpYYQDiOzqkpaoVAsD5RwqIKukBcAMBlXwkGhUCwPlHCogs4gFw4TMbVcqEKhWB4o4VAFYb8LLgdhUjXfUygUywQlHKqAiNDR4sGkshwUCsUyQQmHKukIelXMQaFQLBuUcKiSjhaPijkoFIplgxIOVdLZ4sWEijkoFIplghIOVRIJuDGbUBXSCoVieaCEQ5W0+t2IpnPI5QvNHopCoVA0HCUcqiTi520z5lK5Jo9EoVAoGo8SDlXSGuDCYSahgtIKheLkRwmHKon4PQCA2aSKOygUipMfJRyqJKy5lWaUcFAoFMsAJRyqJKK5lVTGkkKhWA4o4VAlrZrloNxKCoViOaCEQ5Uo4aBQKJYTSjhUidvpgMflQDytUlkVCsXJjxIONdDidSGmhINCoVgGKOFQA0GvU1kOCoViWaCEQw0EPS7E0vlmD0OhUCgajhIONdDidSGRUZaDQqE4+VHCoQaCXpdyKykUimWBEg41EPQ6VUBaoVAsC5RwqIGgx4W4ijkoFIplgBIONaDcSgqFYrmghEMNtHhdiGdyYIw1eygKhULRUJRwqIGg14UCA1JZtRqcQqE4uWmKcCCiLxPRXiJ6loh+TUQR6bVPEtEBItpHRK9sxvjsCHqdAIC4SmdVKBQnOc2yHO4GcDpj7EwALwD4JAAQ0WkA3gpgK4CrAHybiJxNGmMJHie/XJmcshwUCsXJTVOEA2PsLsaYUL8fA9Cn/X0NgJ8zxtKMscMADgA4vxljtMLjUsJBoVAsD5ZCzOEvAfxe+3sVgAHptUFtWwlE9D4i2k5E28fHxxs8RI4uHPJKOCgUipObhgkHIrqHiJ6z+LlG2udTAHIAflbr8RljNzDGtjHGtnV1ddVz6LYIt1JaBaQVipOWe54fxY2PHkEqu7xrmlyNOjBj7PJyrxPRXwB4DYDLWDE3dAhAv7Rbn7ZtSVC0HJb3TaNQnKzc9MQxfPLWXQCAB18Yx/fftQ1E1ORRNYdmZStdBeATAF7HGEtIL/0OwFuJyEtE6wBsBPBEM8ZohRAOaRVzUChOOpKZPL70h7148bp2fOzKTbhnzxgeOjDR7GE1jWbFHK4HEAJwNxE9TUTfBQDG2G4ANwN4HsAfAHyIMbZk1HSvCkgrFCct9+wZxXQii49cthHve9kGdAQ9+PmTA5XfeJLSMLdSORhjp5R57fMAPr+Iw6kaj5Nn1S4n4ZAvMDDG4HIuhdwFhaJx/OG5EXSHvLhgfQecDsLVZ/biF08OIJnJw+9ZMhn1i4Z64mtguWUrpbJ5vP37j+GCL9yLZwdnmj0chaJhMMbw6KFJXLyxE04HjzFcurkb6VwBO45ON3l0zUEJhxpYbnUOv316CI8fnsJELIOv3vVCs4dTNQ8fmMBf/ehJ7B6ebfZQFCcIB8ZimIpncMH6Dn3beWvb4XIQHjm4POMOSjjUwHKLOfxm5zDWdwbxkcs24oEXxjEZSzd7SBVJZvJ4/0934N69Y/jbm3aiUFBNEhWV2TXEFYmz+/VOPgh6Xdi6MoynB5an1ayEQw0sJ7dSRjOnLz+tBy/b2AkA2H4CmNcPvDCGaCqHa7f14dB4HI8emmz2kBQnAHuOz8HjcmBdZ9CwfeuqVjw3NFv3TsyMsSXf3VkJhxpYTm6lvSNzyOQLeFF/BGf0tcLjcmD7kamGnzeTK+Cvb9yOq77x4LzWznjghXGEfS58+jWnwe0kPPjC4lTPK05s9hyP4tSeUEnixekrWzGXymFgKlm3c82lsnjDtx/B6Z+5E/ftHa3bceuNEg41oFdILwPh8MwgN7PP7GuF1+XEpp4W7BuNNfy89+wZxd3Pj2LvSBQ3Pnq05vc/fzyKrStbEfK5cfbqNjy8TP3Fito4OB7Dxu6Wku2be0MAgBdGo3U713fuP4hnBmfgdBA+/stnl2wlthIONSB3ZU1l80veLFwIh8ZjCHicWBXxAwDWdbbg8ETjhcNtzw6jJ+zFGata8ce9YzW9t1Bg2D8axakr+AO9bU0b9h6PIp2r38M3NJPENd96GB/82Q7kloF7cTmQyuYxMpfC6o5AyWsbOrnAOFSnez+XL+AXTw7gytN68N13novJeAa3PXu8LseuN0o41IDDQXA7Cb96ahCbP/0H3LJjsNlDahjHJhNY3R7QWwes6wxiaDpZ14nWil1Ds9i2th3nrW3Hs0MzyNYwAQ9OJ5HI5LFZEw5besPIFRj219Hi+fztz+OZgRncsWsEv9i+fAukTiaGZpJgDFjdXiocWgNudAQ9ODQer8u5nh2axVQ8g9edtQoXru/AylYf7to9Updj1xslHGrE43RgcJr7H3cPzzV5NI3j6FQCayRNan1nEAUGDEwlyrxrYcwmsxiYSmLryjDOWRNBKlvAvpHqzfnBaT42oQFu6Q0D4MHGehBNZXHP82P4y4vW4dSeEG5fohqfojaOafe0lXAAgPVdwboJh0cP8gSJCzd0gIjw8s3deOjAxJKMYyrhUCMiKA0Ax2frF6RaShQKDMemEljTUczcWNXG3UvHZ1MNO+9ebRLf0hvGes2cP1aDMBqZ42NbEfYB4NaOx+nA/rH6WA4P7Z9AJl/AVaevwCs2d+OJw1OIzSNorlhaDFYQDqvbg7risVB2HpvGKd0taA96AAAv2dCBRCaPvSNLT9FUwqFGjMKBT0bxdO6kqqKcSmSQyRX0eAMAdLV4AQDj0WKtw23PDuOlX7oPX71rX13iL8IiW9sR1IVRLZbK6BwfW48mHJwOQn+7H0cn66P17RyYgcflwIv6Izh/XRtyBaYLtOXC0EwSv9w+cFLVj4zOpeF0EDq0e9zMqogPI3OpmlycduwdieoWLQCcvboNAJZkLYUSDjUiC4fhGS4c/u7nT+NN33mkbpNQs5nQit06pYelM2QUDtFUFp+8dRcGppL47/sO4I5dC/ebCkust9WHVr8bYZ9LFxhW/PbpIVz61fsxPMP3GZ1LIeR1Iegttgxb2xHE0cn6aH3PDMxgS28YHpcDm3p4XGNfHbNYljrpXB6v+++H8PFbnsUPHjrU7OHUjfFoGh1Bj942w8zKiB8Fxu+vhRBNZTE4ndRjYgCwstWHrpAXTx9TwuGExyPlQU/E0oilc7hnD89VvvWpJbP0xIKYjGUAAJ0tHn1b0OOE3+3UhcOjBycRTeXws/e+GFt6w/jiH/YuOHtneDaF9qAHPjdvctbfHsCAjTmfLzD83c+fxqHxOH76GE95HZ1LoTts1P7WdnLhsFDLhjGG3cNzOGMV1/pWRfwIepw1xUROdB4/NIXJOL83bt5+8iRjjMfS6ApZWw1A0aUqlMH5ItybQrEAACLClt7wklQylHCokVa/2/C/nP/83NDJ0ctHWA6ymU1E6Ap5Ma699sjBSfjcDmxb24a/v3wjjk0lcPuuhQVoj88k0dvq0//vbfXjuM0DeXC8GEf4035eyzAWTesuJcHq9gCS2bw+7vkyHuWKwMZu/mATEU7pCRnGcbJz394x+NwOfPSKTTgwFtMtthOd8Wh54bBSc68OzSzMAhUu0rWmlNlN3S04MBZDfom56pRwqBExYQqt+sgEdyX53U5bLfdEQ1gHXSYfbFfIq7+2e3gWZ66KwOty4ootPdjY3YJv//HggnzRx2dT6G0txjnag27MJDOW+z6rFem9ZEMHjkzGwRjDXDJbIryFsBmdXZhwOKx9z2ul9gqrIr6GBuiXGs8Pz+H0la245FS+LO/J0ql3PJouuddlRIKDiGnNF+EiFZaIYNOKENK5Qk3JF4uBEg41IiYf4Y8/ovmzz13ThsHp5AlZGHfbs8P4yaNH9Il9Mp6B20kI+43LfbQHPbrL6dhUQk8ZdTgIH3zFBuwbjeLeGgvXZCbjGXSFiq6sSMCD6UTW8pruPT4Hn9uBV5zajWgqh+lEFtFUDiGfccwrNOGw0MyyI1o8aZ2UwbUi7MfIbKou33kuX8CPHzmypBMb9o9FsbGnBeu7RGHYiR9jKxQYJiq4lYJeF/xuJyai1sIhly/gy3fuxX/ds7/svTA4nURH0IOAx3iPbtCu52IUmdaCEg414tf84SIVTVgO56xpQyKTx3Qi27SxzYedx6bx4f/diU//djdueYr7kSeiaXQEvSVr54a8LsTSOaSyeYzOpQ2pf689cyX62vy4/o8H5j1ZziWzCEuafyTg1qrRS2MZI3PcyljfxSfrI5NxxNI5tHiNlkNR61uYhn9kMgG3k7AyIru9fEhk8ojWIZ31+j8ewGd+txtv+s4jGFqC7prJWBrTiSxO6Q6hxetCd8iLw1Lufyqbx83bB3QL60QhlskhV2D682xHZ8iju1vN3Lx9EN/640F8/Z4XyrpWB6cT6DNZDQDQ3y4y85bW966EQ4343PySubXA9JHJODwuB7au5IFKkQ9dKLCGtVcoFBj+uG+sLrnXP3v8GEJeFzpbvPi1FlCfS5W6ZwCuQcUzOd08loWDy+nA+y/ZgGcGZvRCn1pIZfNI5woI+4rnbQvwB3Y6UepaGtP8xP3aGAamElw4mCyHjhYvXA7SayBkhmaSiKaqE+bDM0msaPUZGrMJq2Rkga6lrGY1bF4RgstBeoB9KSGshA2aMF7XGTQIgi/+YS8+ccuz+PD/PnVCWc9zSf79my1OM10tXkzErF2ct+wYwMbuFvSEvfi/Z4ZtjzE8k9TjF+Zj+9yOhhaYzgclHGpEWA5CqT46mUB3yKtrHjOJLEbnUrjwunvx7v95oiFj+Ma9+/Ge/3kSf/6Dx+fVuVSQyRVw1+4RXLG1B687ayV2DkyjUGCIp/MIekuXRWzxuRBL5fTYilkLevO5fegOefGt+w9Udf4DY1F84fd7kMsX9IfUYDlof08nMmCM4Z7nR5HI8M87Hk2jO+TV9xG+/7DpIXc6CN0hb0ls4N49o3jpF+/D277/WFUtQUZmU+gJGYPdvbrLih97NpHFJ255Bk8crq177WOHJjGdyOIfr9iEs1dH8FgVbcYZYzg0Hlu0egNheQmB2N8e0C2cbL6A3+zkisXu4Tk8eaT5rrHr79uPP//B4xUTBqIpfj+FfKXKkExni9dQ41N8fxY7B2Zw9Zm9uPK0FXjwhQnbwPJkPGNIDxcQEfra7DPzkpk87nl+FDMWSlIjUcKhRkTVsNCaZ5NZtHhduuYRTeVw754xjM6l8fCBybovkJPJFXDjo0cAcFfHbc/aayqVePjABOZSOVx9Ri/WdwWRyhZwfC6FWDpnqBUQtHhdyBUYRrXJ0Fw05HM78dcvXY+HD0xaBivNguyTt+7C9x44hJ8/OYA5TYNvNbiVuMCdTWTxy+2DeO+N2/H9Bw8DAMbmUugO+XRhIjJnWizG3dPqK3Erffv+gygw4LmhOdy7p3KcZCyaRk+rUTgIhWBaS+/8wu/34Obtg/jAT3fU1IPqycNTcBBw0Smd2La2Hc8NzSKZKf/+nzx2FJd+9QH8+23PV32ehSAmxm5NQHa08PgTF1JxTCey+PdrtoIITV857f59Y/jKXS/goQMT+PRvnjO8NhXP4M3feQT/8IunARSFQ7iScAh5Ld1Ku4ZmwRjwov4IzuxrRTKbt3StZfMFzCSy6Gixdl/1t/kt3UqFAsN7b3wS771xO9747UcWtSJfCYcaeeM5q/C9d56Lv7p4nb7N63Lomkc0lTWktz50oL4Pyt6ROcwksrj+7WdjfVcQv95ZubYiX2D4+C+fwVXfeBBPSmsy3L7rOEI+Fy7e2Kn77g+Px5HI5BD0lE6yQW2R9WFNOFi5nt56fj/cTirxvX7h93uw9TN34iEt7TSVzeOZAZ5xdPfzo5gVloOk+bcFheWQxa+0eMizgzOIpXOIZ/LoDnvhdTngdlJROFi4B3pbfQbXz8hsCjuOTuNjV25C2OfCfRWC6IwxjMym9PiFQHz+uRQPmv9p/wQcxDXEWoqadg7MYPOKMIJeF/JbofoAACAASURBVM7qiyCbZ9g/Zp/3zhjD9fdx6+wnjx3Vr91COKplfNkxFk3D7STdUusMepHJFxBN5/SxnrumDVtXhquyfBrJzx4/hq6QF3932UY8cnASY9Hid3/bs8PYfnQav945hGOTiardSp0tXkwlMiVWwS69tX2kbC8voUDYVWH3tQUs3cQPH5zAwwcm8YpTu3BoIo6bn1y8Zo9KONQIEeGVW1cYNGuPy6FPatFUDntH5nD6qjCIUHXDLsYYbnjwIL557/6y+4kUzrP6Injl1hXYfmS6omvplh0D+OWOQewdieIDP92B2US26FI6rQdel1PPmDg0EdPcShbCQdt2XJuIrR6okM+N89a24/69xUV2svkC/ufhIwCAX+7gN/fgdEJfUW9gKoG5JP8MBsvBr7nqkhndhbH96LSeNdLZwoPmYZ9bj4NYuQd6wkbhsP0oF5Av29SFi07prDiZzaVySGbzJcJBWC2zCV75OjSTxD9cvglEqHoFOsYYnhmYwVna8pRrO7lFWi6t8dBEHGPRNN5ybh/yBYb7980/QwzgLrZLvnw/vnLXPtt9xubS6GzxwqFVEQsNeDKWwYGxGIh41s1ZfRE8PzzXtLhDMpPH/fvG8LqzVuKyLd0AYIiB3bl7RF/u9969o4imqxMOEb8bjKEkRnVgLIYuza28sacFLgdZFkaKeEWnTeC7v92PuVSuRND/ascgIgE3vvvOc3H26ghuXsROwEo4zJOAp+iT97qcCHpcIOI3z+GJOLasCGNF2Gfb/uGh/RN6phMAPHlkGv95x1587e4XMFYms+a5oVm0Bdzoa/PjJRs6kCswPFFhhbZf7RjiXUQ/cjGm4hl88c69uHfPKOZSObz2zJUAeFDMQcVirxaLmIN4gIZnkwh4nHpQ3swF6zuwbzSqm8C7hmaRyRXgczt0y0FMfuevbcfgdFIPOssxBxH8T2byGJlNwekgzCazunkvxhP2u8u6lVaEfYhn8vqDvePoNPxuJ7b0hnFabxiD08my5vq4pnmaq6/dTgcCHifmUlndt33hhg6c2hPCM1X2ypmIZTCXymFTDxfO/W1cOJRr+SFW5PubSzYg6HFiZw1WynNDsyUB7+8+cBAA8P0/Hbb1l4/HeIxHIDTgyVgaB8fj6Gvzw+d2YlNPCHOpHMZs0j4bzfajU8jmGV66sRNbV7aixevCU1J68N7jUbzxnFVoC7hxYCxWdCtZWMEyQmkxT97HphJYo7mYvS4nesI+y2yzybim0NikzIrvXQ5KC2v00lO74XU58arTV2DvSHTRig8rCgciWkNEndrfFxDRx4joDY0f2tLG55KFgwMOB6HF68JcKoeZRBbtQQ/62vyWpuJELI0//+HjePlX7tfNTaHNAjDUCjw7OGMQIkcnE1jf1QIiwrY17XA6qOzyndPxDLYfncIrt/Zg68pWvOeidfjfx4/hwzftxKqIHy/bxAuaHA5C2O/GTCKLuE3MQWwbnkmV9dEK83qf1mlSmN5vPW81JuMZxNI5HNMmv4s3diKTL+jtA1oNwoFf44GpBHIFhjNWtfL/tWsqXF8hH7/u4m8zIogq4g57j0exuTcEt9OhLwy0v0z7gqk4nxCs0h3DPjdmk1k9KN0b8WNDd0vVNQBCqAjLLahljpXLXDk8kYDHydc73tgTqrqj52wyi9f890P4l988h0c0d2ehwNuCtHhdyOQKej2HGXMVcYd2LSZiGYxHiy43vedUk9qKPH5oCk4H4by1/NlY1xnEYe1em45nMBnPYH1nC1a1+TE4nZQC0uUth3LCQV4kaGXEZzl5i/qgDlvLgR9Dni9eGI1hMp7BBRs6AACvOJVbQkLBajRlhQMRfRrAfQAeI6L/APANAJ0APkJE31iE8S1ZHA7SM5dEM76wz42JWJqnZPrd6G8LWFoOv5f88TsHuFbz1NEZrOsMIuRz6Z0+ZxIZvO76h3HZ1x7QzfSRuZQ+2fk9TmzpDZXVHJ8ZnEGBARdu6AQA/MMVm3BKdwsKjOFfrt5iaDbW6ndjPJpGrsAqCIekZbxBsEVbWvH543yCGJxOwOty4Nw1vAPlwFQCx6aS8LudeJHmThGTiaz5iz5WYqI9s08TDlrgzq9Zb7KgEt+JjJi4xAR+cDymT8ZCOJRbBnJKE+AitVam1a8Jh5kkHAR0h7xY3xnEwFSiqh79wu0oYj4AdzHIbqW7do/gDumeGZhKYFWbH04HYfOKEPaNRKty48jus19pactHpxJIZPJ487l9AHgVNMCvx1/fuF1fjS+ayhqus8i6mYilMR3P6tfmFG2pzWa1Fdk1NIuN3S36vcpTbvlYxGpuG7qD6IvwbKu5ZBYelwNeV+l9I9MaKBUO+gpyUkp3b6vfsmpeWMZW9xBQtBzk+WLnMT43nLe2HQC/tq1+tz5nNJpKlsPbAGwBsA3ARwBcyhj7JwBXALi8wWNb8ojJSfgwQz6XblKG/W6sjPhxfDZZkm749MAsWrzcDbVrkD+MhydiOLUnhFURv36D/PZpnomULzA8dWwajDGeKy1lzZyzug3PDMzYugOe1wTNaVodRovXhT/83Uvx6D9dhled0WvYN+J3Y1irJA56LFJZtQeOCz97TWtVxI+Ax4mDWqOxoZkkVkX8+uJBx6YSGItyIRfRHjqhWcmuKoeD4HEW8783r+CfQfwv0m3lsbicpZ01e6T2B9FUFmPRtC4cVkX8cBAwVKb7q0ghbLOyHPwuzCVzOD6bQlfIC7em0RdYdWtRHNXqZFa2GtujC4H03NAs3veTHfjgz57S3WIDUjHVxp4QphNZvSFeOR47xPthvXhdO/aN8vtCWHevPasXTslf/uudQ7j7+VF87e4XAKDEmhTKQTSVw2Q8o8cgOls88LudlkqRKJQ70qBCOd4ccRanaxYmwNudDE0nkckVdEG8TrccEphL5UrSn62wshyOz6bAGA8mC3ojPL5lfuZjmoVilTAB8Pso4HEamvsJi064rYgIZ/VHanIjLoRKwiHFGMswxmYAHGSMJQCAMZYDsLhJt0sQoaUKrSPkc+mTTNjnQlvQgwIrpssJjk3FcdrKMNZ1BvHcMHe5jEfT6Al70ddWzB8/IC1Ss28khplEFulcASukieTs1RHEM3nb7Jbnh+fQ1+Y3aPoup0O3PmTCfrc+frtUVn3fMm4lIkKvlD46NJ3Eqja/wa86l8oh7Hfr55mKZ+B0UEnbZK/LgRntgdQrSc1uJakq2knlhEOqRFN3OR3oDvkwVKbjpqh6by9nOUh9oUS687GpypPg8GwKva0+PdALcPeVEA6PSzUTj2iB1WNTCd0NIYSEXYNCmd1DvDfSGatasX+UN3oTWu66zhZ0tnj070y08RAN4cxJCkIhSmRymE5kdJcbz9m3dqde93teKPeh/32qIfUZ47E0JmIZvSAVANa0B1BgXEERQeGesBerIn6euj2b1JW8clgJBxH7kmMxK1v9yOQLJcI6ls5pmXXWU654ZuQ2L7uHZ3Fab9hwb5zV14oXRqNIZRu7XC9QWThEiOiNRPQmAK3a3/r/Cz05EX2UiJgU0yAi+iYRHSCiZ4nonIWeo5EIzdWjWw5uPRAX9rv1tD9z87gjkzyItbo9gJHZFFLZPOZSOXSHfdqDxXs0HZ6I44xVrfC4HDg6GS/6taWJ/ex+7qp56qi1NnFwPG5oEVyOVr9bv6mtArvyjV3OrcTHWDSvh2aS6GvzIxJww+UgTMYzvFWGz6WfZyaRMbRDF3jdTsxok3OfLlyMbiX54XZY9OT3e5wI+1wYm0uV+PgB7icu13tpOpGB1+WwnETCmnCQ+/OIyWIiWll/GplNlmRB8Z5SvIbgqaPT6A554XE68NTRaaSyecwksrr1KCyOalpuiNX95EZv41G+0E3E70ZnC8/lZ4zh2cEZhLwuJLNc8cjkCwa/vHCrjs6lkC8wg7tE3MMyyUwev9LWXN89PIddDehgLNp5yN+tCABPxdOYjKXhczsQ8Lj0+3cylrGdsGWEMmQQDlHRvbj42YW7bcpCOFSKa6yM+PU0ccYY9o1EdRetYFNPCAVWfRbkQqh0VR4A8BoAV0t/i58HFnJiIuoHcCWAY9LmVwHYqP28D8B3FnKORiM0KaFFeV3GyVO4TGakfkuJTA7j0TTWdga1RnbpYhfUkBd9bX7E0jyl7fBEHOu7gljdHsDhibjedlrWVNZ0BNAe9Oj+STOD0wn0W/RzsUKe8AMWwkGed30VtK0VWm1BLl/ARCyDnrAPRISQz4VoKos5zYctrmE8k4fbwiUkX9P2oAdel0OfCIXlIFsbVpYDoKWzapaD00FGP3HE2k8smI5nbHvvBDxOpLJ53vRP+yxCSMj59XYcn02VtFRoD7qRzTPE0jkcHI/hzL5W9EZ8GJ5N6ZOTKBDsjVTXWDCVzWM0yv3jQqCMzaUwEeML3TgcxZbss8ksUtkCLtVSQUXmldnV6Pc49e9CniD72gIlAfWdx6YRTedw3RvPAMBTkmX2jUT1Qsj5clQ7p7z2uQgAT8YymIpn0BHk341ssVopJWZ8bgc8TodROGgCQO7oahe4tisslVnZ6pcWrkojnsnrMRyBUPTK1cHUi0pX5TkAu7WfXaa/nyvzvmr4OoBPAJDty2sA3Mg4j4FbLr2W714CCK1XWA6y1hr2ScJBulGET7GvzY/OFi8m4xl9EukOeXUNbCKWxvBsEms6gljbEcDRyYTut5Rz+YkIZ/dHsNMidXI2yTuVyj7RcojxArCcqKuZhAW9rT6MRVP6ZxcPTcjnRjSVQzSV435Wt1NvReKxCAp63cVbNOBx6tecqJjqKo/LynIAuLAanUvj4HgMa9oDhhX9Vrby9EO7oO50IqNPxmY8TicyuQIvHNTG5nNzS8Wq3YJMocAwKiUYCPSeUvEsbxMS9mFF2IfjM0ld0RDfVUfQA4/LUbF1OP98wOoOvy7opuIZTMSKLR06W7yYiGZ0d4mYiOxcjX530UcuWw69ER/mUjm91QkA/f686vQVWBXxG9JLdw/P4pXfeBCXffUBw3tq5egkF/yysG2TPqscGxH30awWkK4EEc/mE/U4QNFykBWHokJoshxSOUtrXKY34sNELK3FR0otXIAH2O1qKepNpavSov2cC+ADAHoBrATwfgDzdvkQ0TUAhhhjz5heWgVArvIY1LYtScwxB5csHPwutGpFXLIWIWctdAQ9SOcKODzBNZ7ukE83PblriT/8XSEuRERA0hzUOnt1BAfGYiXayqBNDyQ7ZMvBavKXu7TazME6PWGfwfwVAi2k9WfibiU3HA7SLQCPhUDySdfW7Sz6bHldCd/fILTshIOWf35gLGbIDAL4dc/kCrbdVWeTWd1FaMbtImTyBcTTeQSk2hB5YSSACwKzJTEZzyCbZyVuJTHZjEVTmEpk0NXi1ZIbipaD+K6ICCtbK68rITT5/raAvhbJRDxjSFHlykpad42KiWnQpujR73HqNTmywiJabIxJ6x88PTCD9Z1BRAIebOk1LpL0w4d4S5TxaBoP7CsWT9bK0ckEVkX8BjeRbjnEM5iMp/X/hUs4ls5V5VYS75GF10QsjbaA29CMcaGWA9OWI9XdnybLweNyoL89YJtyXE/KXhXG2L8xxv4NQB+AcxhjH2OMfRRcWKwu914iuoeInrP4uQbAPwP414UMnIjeR0TbiWj7+Pj8b6iFIG4qod3KE6psOcxKWoTQ/NoCHn0SEIHn9qBHf8iERhb2uxD2uTGXyuqFWuaHVCxSbi68sltcxA6PKVPIjLzJ3M7bjNAkhYASGSEtXhcm4hk93Rcojd3IiGsrBLHbxc8r+//l625n0WzobsF4NI39YzE9fVWga3txa7dGzKZiHAC8TgfSuQIy+QJapJYj8sJIAPCp3+zC+Z+/Fzc8eFDfNqW3VDBaJULb3T8WA2O8+E4E+Ke0YipRPQ6IAHZ5K0UEY7tC3qI2HeNWQqe0gFU2z6SsniCcDrK1HAIeJ+JaDyjZ/SfcnrJwPDAWw2bNf74q4jfESHYem8Flm7vRFnDjbm3J3fnAYypGK9nndiLgcWIqnsFULIN2za0ka/FWVrIVfrcTCann1WSstJFeuIxwCFVhOQA8VfzgeBxBj9PgQhas0TwJjabaCukeGLOTMto2WxhjlzPGTjf/ADgEYB2AZ4joCLjgeYqIVgAYAtAvHaZP22Z1/BsYY9sYY9u6urqq/Bj1RdxUYlKVJ1Svy6FrEXLMQVgOkYBbv7FE9o3f49QnfrEkYavfjbCfr2sgHnBz36Mz+1pBBDxlijuIycm8dKYd8vitFPBqNHSBsG70bqmSW0n4VYXAEJOOlQYnJh0R4yhaDtZBaIfNHX2KZJ6LIj2B7max6XoZt6kYN49ZjtPIXTyPTSZw0xPcIP7mvQf0icMu913cN0Jp6GrhwiFXYDioTdyyC7A96MG0jWATyOm4bie/N6fiaS3TyG0Yh37ekBdhn0tXMszCwec2FoIK9JiLZjlk8wUMTCWwTltFr68tgKjWKmI6nsHhiTi2rW3HOavbFrTU7tHJUuEgPtdUPIOpRNGtFDQIh+qmwYDHaWiIOJ3IlKQ3h7QU9TkL4WCXxioQ2W7HZ7nlsKG7xVIJW9MewLE6rIteiWqFw40AniCizxLRZwE8DuBH8zkhY2wXY6ybMbaWMbYW3HV0DmNsBMDvALxLy1q6AMAsY2xhCxM3EHFTCY1XuJUcxDVrt9OBFq/LsADQrOQzFpOSePj8bkk46Cmxbn0SHZ5JosXrKpmYQz43Tu0pLYYrV7xlhcPgNrKyHKp3K4nJVK/70Cwi2RcvrKSWssKBH0e0KxGCWF5Ny1lh3AAMgb3TTMIhUmbdCKA0x19GtnZkAdLqd+uW3p8OcMv2S28+E7F0Dn/az/+fsREOcmU4AHSHi91nxb3SKgkHkd1Ujql4Bi4H6dprR9CDiVgGqWwBfu1aCmtscDqhZzBFAh699sWs+cotZDwWloNwow1OJ5ErMKzr1GpLNEt2aDqJA5r7ZEtvCJtWhHBoPF5V8aCZ2UQWs8ks1rQHS17raOEL9aSyBV2xkhUsbxUxB4Dfc7JbKZYurZFwOAitfrchzgiUv4cEYiGp4dkkDo3HS+INgjUdQUTTuZKMqHpT1VVhjH0ewHsATGs/72GMfaEB47kD3LI4AOD7AD7YgHPUDZdpMhMarDx5m/2U0wn+kLZ4XbomMRFNw+UgeKTurmISCGuWA1AUDlacvTqCpwdmDPnjU/EMQj5XVQE3oEbhUMly8BpbaYtCNdklJv7WYw4W4/SZ3Uq6cJDcStLb7NxK/e0BvPqMFXjHi1djbYdxAjG33jbDe01ZX3eD5SBNOC0+l17f8vihKawI+/CGs1ch4HHi8UO8dkEoDaL7rMCnXQdhdbUHPPo1GtYqsWUXVnuwsnAQQXWhiXa0eHTBLa6t+D08k0LE79YnOqGgWrmVBPJ31xbwwOUgPXYhKpTXaU0FV0WK6bfiM66M+LGppwW5ApuXP/2oVlOy2sJyEG4loHg/yeuVVGs5+D1Gt5LdfSFqX2TkbDY7Ah4XIlrPp6GZJNZ3lgo6QG610dgeS5VLAzUYY08BeKreA9CsB/E3A/Chep+jUYgAalbrLuq0CJB6XU6kJU1oJplFJOAGEekay2wyqz+YIclKAITlwCePoZmkba702f1tuOmJARyeLGock/GMbS8XKwyTbIWYg52GLtDdSjPGgKUcuBRWgZh0LOsctH18unDg55WzmIxuJetxOR2Eb7/jXMvX2gLF1uBmcvkC0rlClZaDJPi8Lh6L0PoVbexpgdvJW4iIAjM7t5L4rKI+xu9x6sHu4ZkkwtrELYgE3EhlC0hm8rYFXVPxovsI4NdcxBb82rXUz5vI6H/L7iuzhm10KxndfF0hr+5WEpOYKIIUrp3peEb/jCtafUhry8EekmpzRmZT+NubnkImz3D9287WJ0YzVqsTyuPUrXPt+ricDnhdPF5UtXBwO5GUis+iKWtXkRAO2XwBB8dj2NjN60qqKbbb0NWid0Ywx8YEvdK66KKbbyNQXVkXgLAcdOGgPbAuyfHtdTkM1YwziYzuUxYTSyyd028ct9MBn9uhF8OE/S5d6z4+m7L1W569mt8kchO+qXi64tq4MrJ/02rud1QR+BWIiXJ4NgmiokvCWEgl9i0TkHZZWw7yZFRpLJUI+9xwkLXlEE/z785WOEgTi7yP+PzxdA4DUwk9nXhjdwiHJ/jaCTOJLHxuh2GSBYqfWcSqfG6HbjlMxjMImPZvr+AW469lDULI63Lobi1zMeFMMqtr2IYMNpPgtbMcAO5aEm6l47MpuBykx9jkjJ7jsykEPU6EvC49pVfO6vra3fvw5JFpPDMwg4/+0pzcWERUdlvF1/hn1a6ldN+Uc2daETBbDqnSNcsBTYhk8vjBnw7jqm/8SV8ZsJrzbF0Z1lvhvMhm4i9aXgtbnrYSSjgsAJduOfAv0ynFHAQ+t9FymE0W12eWJzhZqxA3nNvJq1DlVhV2yxme0t2Cla0+3COtajYpZWdUgyHrx8pyqBCwlhEPXjSVQ4vHpb83YJFlVAxIW6Syuo0Tl5iEZC22UnC8Eg4HIRLwWAakY5pL0DYg7SqeW/5sLb7i8qXTiayu0a7tDCCZzWMsmsZ0PGMZD3I5HXA5SHdL+dxO3Q0yk8iUTMQiZlLOB20+l9fl1DvZimssBEIik9fvTbmRoTk4Kr9mtvq6Qj49tnR8JomecLFFSNDjgoP4Ikkjs7zOg4jQEfTA6SB97Y1kJo/fPTOMt53fj8++9jQ8cXgKO45adyAWixG1BUqfD5/bqcd/ZItTt1irdLv6pYB0OpcvqRrXr4XLgUy+oHfLFWswVJMVJeJhbieh2yaRJBJww+d26OuqNAolHBaA8AmKPHXdcnAaLQd5ychkJq/7puUJTn7QRJAr7HPrxTcCu0mKiHD5aT340/5x3VKZqtGtJGf62LmNxOZKqaxOqWutzxAfKJ5ECFdzMaGMveVQP+EAcIvGatEksc3ecijVROW/ReND0RdKxDsOT8QxIykKZsSELeo7xD1TYKUaaJtFx1Azwp2pj9vi3pPvQSEoyikEfinuYZ74usPFbC1eBV6c6ER7+NkkX29daPsObb3vUc0d9dzwLFLZAi7b3INrz+tH0OPELVoLDjNjc2l0aYs/mZGtBdlKK7ozq7t/Ah7uVmKMFRvpWdwXXpcT6WxBd6uJleFcdql0Elec1oM3n9uH/36bfRkZEWmtNpRwWLJcu60fP/7L8/HGc3idnphQ5YnV53YilS1aDulcwbLdhmw5FDU5o08esPbLC166sQupbAFPa/UOc6msIaulErVkI1UzIQsXmFwc6DJMNkbhYJmtpE1SYvLRYw6yW6kOwiHgcekuJBmhvdsJB3lSlIvghEYpKl2FAiHSOY9O8uVY7QLdegqvxT1gvk5ByYVlh6yUyMcHJLeSRQyhXJypnFXRrRVuZvMFHJ9NGZpFAkW/fDSVMwjInrBPdyuJup2z+iMIeFy4dEsP7tw9atmBeCyaQpeNpi1bC/KYhaJVvVvJhXyBIZMv6JaItXDglsN+LSVY9CurxnLoaPHiK285C1edvqLsfr1VFD4uFCUcFgAR4ZJNXfqD4XIYfwOllkM6V9BvViLSJ3v5ptXdU9q34zTEAuxvsPPXtcNBvDVzvsAMqXvVUClbybhv5eOJOINdewuhSZUz7y/Z1I1IwI0z+rj/1Zw+XM1YqyHocVq2boiXmQTM47CyHMx1HqIGYCKWQTKTL4k3CMyuHtll5TZdJ/Ga7A83k8zm9WMBRuEq3u+VhYOwHMq4Gu2sWKBYJT0eTWNkNmVoMw9w4TCXLF1Yqifs1d1Ku4fn0Nvq06/Z5Vu6MRXPYPdwaS3EeDRtWTAGGK0F+W8hLM3X0w7xjCYzeV1psIoBel0OJDN5vQ3JpPbbnN24ELolt12jUMKhjlilsnrdDpPlkDc8mEKDkx9+8XbxYMpzX7l5sNXvxrrOIPYcn9MnumCZB9iM00Krt6NSKis/dwXLQQ9I22crnb+uHU//65X4q4vXGfapt1sp4HXp1b4y+nW0EbJiPEQmrdSUdSYsCVGxOx3nNQZ2wsFryiByOx26IDK7QYrNC43C7ZO37sJnfvsc0rk88gVmtAwkQeGzcivploN9kkK5vH0xoe8b4R1dzf2jxAp65nTQDmkti8HphCH76EJtRbRHDpauzz1WRjjI94osIMvdd1bIQljvVmBxDTwuhyE5QBg6rjrcpwJRgd/IQjglHOqI02Iy97mcRsshWzDcrOZJAJCEjIWbqtKkvb6rBYfG47oWGajJcpA+i82NTPq+lW908TntLAexvZbAoLllCbDwbCVAsxws3DJCsNulIYoxy72egOKkISwHOZGgTQt+p0zavIzPlMIrxgiUukHEpCVX7yYzedz0xDH8+NGjeFhbElT+DFbxLrezuJ6GleVg/s7LCQcxUQsXZ6+NWymeyRsUmJBUHzI8Y+xY2x3yYX1XEE8eNgalM7kCpuIZ3VoxY2c5lGvbYoVfFg4VLAfx/MkCq1r3VTV0tXiRzhX0pIJGoIRDHbFahazUcjAJB1MFMFBqMRj9vuXHsL4riKOTCb1JXy2WQzktUcD0fSsfT1wPORBnZUUEa/D9ikOZ8+oXCq9+LbUcRLWunb9Yb+dhus6ilcbIbAoOMrb7aAvytbpT2bzlsqaArDSUFtmVCgcRcyiOf4+0rvT+0Zh2TOu6BDHpEUlJBFXEHMp1Ge0O80nx2UEhHEyWg5+vfZI3LUkb9rmRyfOajZE5YyAbAF7UF8Ezg7MGjVlfdCdc2XLwWwSka+mtBHDBK+odAhZKgyxseiXhZjU/zBfxWRvpWlLCoY5YadM8cyGPqXgGc6ksMrmC4SH1mLJxgNJiOkP9AcrfYBs6W5DJF7BvhE8ItVkO5VNZ7fa1QwgFO3eVOSBdjQYn5gSP5CFgIAAAIABJREFUwa1U8W0VCXqdJW4ZAMhoNSx2rgfZcpAREw5Pd3QbvkPR6yeVs3cr+SxSSe2uk9NB8LkdhpjJbqlHkejwaQw4W0+YQhjpzSTLZCtZTYyCzhYviIBnB/k4ek2TvN/tLAb7pWsn3G8Hx/kKdGaL48y+VkzE0oZgrKjEtnUr2bjTaq1zEPvlCgVJabAv3ARgiLVUk61ULWINiUYKh+pnDkVFxH0iz5tetwNzqRzO+dzd8LkdXDhID6aYdOR0z6LFINxKxeNVur96tJtRtCCwWgvafvy1BKSrEA6mDCPAaDmY3UrVPKQFTToYYw4Lf+gCHhcSFtlKosDRTnAVLQfjoyS7usy58G0BD45NJbSAtI1bycLdqNd6WFyngMdlEG5DMym4nYSwz6036zMEpC1iDvLf4vqSZMWakyHKWQ5upwPtAQ8m4xm4nYROU72NfE9YFQ+KxWzMloNYH3rvyJzuchJtw23dSi7rz1prnYO4X/MFpt8XVveswXKQhFu1Fko1dFl0vq03ynKoI3aWgyCVLaDAjBObmOzkFszmlFhj5XL5G0zkvItAqNWKbnZUG/gGqgsCW1kOVn+LCaGaBmgiuFfPCmmAC9FMvqA/9IJyGiIfR2lCAWD8nObCxbaAW7McyriVXE7Db8AYGzAT8DgNwk0Uva1q8+vptFapqvJnAErTqM3JETKVGsmJCWxDV0uJ6092sbQYYg5abzFtKdhWv7FOR7SGkZfJ1C0HG7eSzyLQLo+/WstBKDY5STiUq80BjO60emYr6c0iG9h8TwmHOmKVjWClGcoPpvBzy5Wd5lRWQH5Iy49BCBnRVK2c6W+mlpXeqoo5WLQTsRIOQrOuRjgIt5I8udTHrcTHYI47lNMQ5e1mLZqISj6foDXgQTSVA2NGl4eM3nDQU2o5WI0laIqZTGpLm3aHfHrPKDu3kqxwiO9MvG5lvernrNhllGvNGy3WMJfvCfk44lod1xcRMlldQb4OirxY0Fg0DSLYFnzKn9VgsZSxxKwQ32ehwPSuB5Ush85QcUzuOmYrRfR+YEo4nBBYadOyIBDIN4/wE8vLUJrdSvLfldw5kaDJcqhBONSSFVXJggGKE3glyyES8OBLbzoTrztrZcVjikBkLWOtBhFQNtc6ZLTWKPYBab7dykITn8/c1tln49Ix7iMCwxZZRRZCNGCKmYi+WoYMJRvhICNateiWg8P+vqvksrxkE19nxWrylZ8BS+Fgs/ocwDsTHJQsh/EoX+HNTjOXr7F839ZsOThly4HfF+WaRQLG57qeloPb6UDI5zKsFVNvlHCoI+JBkoPG1pZDcVu8nOVgkT1UaRoMaes96Kt3zbcIroKWU51bSbMcnDbCQTrftef12/aSkRE5KvLZ61UhDaCkSjqTK8DjdNgKQzHJWRWEic9vTgqQJxQ7t5KY0ORiYLHNOuZgbAo3nciiPeixDTwLQWH+WF2aa0ZfF13cdxYfv9Jk95Ztfbj6zF68/5L1pe+VvjNjN1vjSohWcY0NXS26qwyAYTU7K+wEYS2JEEAxtpWvwa3UbhAO9bMcAB67UpbDCYKVW8nKcpCDgcKnLXdPtTLlySL+YAURX6RFCJ1ATams1n9X2tcOMXnIk7dVQLoWRIzG4HKri3Cwthyy+ULZQGJxfQl7y8GsmXoMPn7rR/DVZ/QCMNXMmFqJGMdv7A01GePatF0BmNhuLuLq0YK6omZC3IvzqbUKeFz41tvPsXYrGdbBKG07IvoGWSk3G7qDmIhl9IWz5HWwrRDX27xPf1sALgeVBL3tkGMOmVwBDrK+h81rWwjcdcxW4sd2W7aZrxcqW6mOWAeky5udAtn8FMexSiOsxoXSGnBjMp6B00FV+1PN56tPKisZfgPWRXC1ICapWtqHV4OYrHKmvj2ZXKGsZulxOmzX+rX6/IBRWNhZDueva8fNf3Mh1ncFS/a1Go+81kA2z4uj2oIezCWLAsOqCC5savwngroiyFvpO3I7CZdu7i67j937BFYupmiKV05bCf712opyBydiOGd1GyZiab1nlRXimG8/37js/eqOAHZ99pVVrbMAyNlKBU1pKJ+kABhX7Ku35VDNCoALQQmHOlKsSyhus6ugNCN3zCya8lYxh8rj4NpKHF6XvTvECqvz2VFLKmsjLAerIOpC0LXCvFE4lJsEAC7sbvvIS/XGejLCDWGeFOTj2cUcAC4gZITFaZUv73KS3pBOrF8c8bsN7eKt+neZM6ku2dSF636/Fxeu560qil14rce4//Ovth1/OQxrnkidbWWhYbew1QZtydeDYzGc3R+paDls6gnhzr9/GTb1lC67Wa1g4GOWLIe8vdJg1fEWqG8qK8Ath0MTsco7zhMlHOqI1WTXbtGvX7Yc1nYEcGQyYZgwxHGs1kauZq4XZnqtGnVtvZUqH09MAPJEUEuhnRXCcpDfWQ+3klN/8E2prGUmAYGd1iomk3JuJa+NW8n6eHxfKw+Pk4rCIaUJBJ/baZmmChTbgrSblijd0hvGvv+4SurKOn+3UjnsLAeR5ZUvMNs6ir42P5wOwtHJBGLpHNK5gl4UZofdqmq1INc5iFiUFfLzbazxqa9bKRLwNDQgrYRDHbEUDhbpdfKEcOsHLzKsfAVIVoJ0L5HptXKISclZo6ZiFEbl961pHLLlIAen5+EO+shlG7FvNIqXbewqHqcOwkE8xOZ20OUmgUo4LT4/YGycZ+dWssIluTVKXpMsh7TmXuLCwTpD6ZzVEbzzgjX44Cs2lBzL0JqkvsqujiwwzRq1LhxsLAe304GVER+OTSX0CmE5ZbRRiMk9l+cB6UqWQ8DjNFq4dbYcWrw8zsQYq8lDUC1KONQRvfGetM1SOMjZDFretoxV+mDRvK98E+jujBqfbPnQ9XErlY6j0mpzlTh1RQj3/OMlNY+lEuKamWMOldxK5dB7S5kmBY+NNl8Jh+TWMCMmVKBoFXhdDvt6BqcDn3v96ZXP2YBJh5/fOBbDaw5CBvarHgLAmvYgjk4lMBHjPveuluqCygvBKSkQmTJrT4skk3PXtBm219tyCHpdKDDer62W+6haVLZSHbFyb1jVGVgFpA3HsQg+FwVG5XGIe7bWB9uue2qlfe2olMpaL22nHpaDfcyBVZ3qaDcuc5ZKtTEHuzHm8xbCgUgXGqILsM/tlNYOqWHgErW4M2uh3EQprpt5rWyZ/vYABhbdcigK52ye2cYQzlkTwbsuXIOvXnuW8f11thxEbU6szCJPC0EJhzpipalbTYDlMisAuVV3cVstD6lrnpZDLcKkulTWUguoHhO5mXpkKzlNLptMroBP3roLB8dj8w4kWmWdAdXVOZQdo0UAwOlwlFoOboeuiMw3jVIMvZExBzN6rKaMUF7TEcBUPKP3EKsUc6gH8j3CY1H2rU/+/ZrTS3o91TuVNajX5jRGOCi3Uh3Ri+BsJqu3nNuHF6/vqKiJksWkUksqa7H9Rq2WQ/X7VlUhbXH+RgiHejxzbqn6FQAePjiBm544BgCWmUjzObb+fxV1DlaI61mwcCsZYg6a5eB1FQPS89VaG/F9AeUL6Kpxi67RFgHacXQaTgcZ6gkahbnOodq1p/X3N8hysFreth4o4VBH7DRYt5OQzTN8+S1nWb5uxiqVVUQyapmUa485VL9/Ndq61QTQEMuhLtlKxWAjYGxoNl+3kmj1Yb4Onnm6lZw2tRiAdczB5y7GHOZ7jRoR6ASqsxzK3b/9knDoCHrqkrFWCXNX1lpjUfUXDtYrANYLJRzqiN3N/NSnr4DF82yLVSprsd9S5fdbLVda1XlrcStVlcrKjye7JBohHOpa56B9UaI3FVB9Y7ZKx9aPZ1jsqfpjr9fckZssKo55zIELBYPlIC0zOh8aFZAuNx5xj5SzLlZ3cOEwm8zi9FXh+g7OBuGuTecKZbOV7Ki7W8mr3EonDHbaS7msCyuEtmZ1L1Va7AewTiGthtpiDtWPg0mZ+fXO2Kh2LJUQ1+rBF8bxxOFJg0Cb78QqDlEiHKTj1aKZX3RKJ3734YtwhramgYzTQSgw4LX//ZCeGu1zO/TGffMVynXsFWegnEAvriBov0/Y50Z7kC+atLW39Ho0AnENv3znPgDFxoLVUm/rJmjTD6xeKOFQR/QK6QUfh/+2mvSqy1YSwqG2J7ua3a3aV9hh6VZqgCZal2wlbUL63TPDAIzVyfN3K4ljm7KV5nk8ADizL2K5XUyku6QV4AyWw5JzK1VjOZQ/twhKn7ZysSwHU+yoUZKzSvSYQ4PcSipbqY7Uy2VSrj13NdrHfGMOtWjgtaSyGtxKdfa7AvWxHMwWzYS0/OJCJ4FylkO9sLov5JjDfNtF6xXS8x+aJeUmfrvKcjMfu/JUALVr8PPFfI1rcQk2gkZnKzXt0xHR3xLRXiLaTURfkrZ/kogOENE+Inpls8Y3H+qlFZfro1TNKfSaiJqzlSrvX6nXjmEcFjstVcvBfAx5+UWPa37HryYgXS+sF5pyWtaa1MJiVEibERZvpe/1olM6ceS6q7G2Qmp4o6h3r6RaOSljDkT0CgDXADiLMZYmom5t+2kA3gpgK4CVAO4hok2MscY41eqMfjMv8J4pts+wcis1Mlup+n2r0ta1XWStc6lmK5mvVTRVfODmO5mLz22eRObrpiqH+Ro4iH8msSjNfIOh9XKVmik3HnG567lyWiOYSTaur1E1eFwOeJwOvT1/vWlWzOEDAK5jjKUBgDE2pm2/BsDPte2HiegAgPMBPNqcYdaGXqi2wOOUizlUc+xqNa+S99XZrVTP95WjLm4lCy1wVcSPF69vx5vO7VvQsc2fuREap1m4+dy8r8+6ziB6wl586uot8zpuo2IO5SyZYvHg0vZ6H5tKVLXff77hDDx+eLIhY/juO8/Bmo7GWE7NEg6bALyUiD4PIAXgY4yxJwGsAvCYtN+gtq0EInofgPcBwOrVq612WXTqlcesPxxyKqvptbLjsEiFreW85SgGpGs6tHSO+b2vHPVJZS2diLatbcPXrn3RvI+pB6RNx67ncpECswAS/nC/x4nH//nyeR9Xr5Ce9xGsKfesWK0TvtT4s239eOeFa6ra9+0vXo23v7gxc9Slm3saclyggcKBiO4BsMLipU9p520HcAGA8wDcTETrazk+Y+wGADcAwLZt2+p9786Lek18xZhBcZu+PGZN2Uo1Coca9q9FW5cD0o3QROuRImh1iIUGokUK72L4ps1adr0asTUiRgSUdyst5nWbL+96yRpsXbk4KbTNomHCgTFmq64Q0QcA3Mp4xO4JIioA6AQwBKBf2rVP23ZCUK/+M+VaZdTSPqPW57qaOZaIf85qxrFYj3Y9XFVWQqtesYFGtaCQKVdotxCoTq5SM9VYBUvZrdTsTKXFoFmf8DcAXgEARLQJgAfABIDfAXgrEXmJaB2AjQCeaNIY581CteNi4z2LmEMNlkPN563ifbXUOejvqbtTwkijtNuFZhWJa7UY+fBm66kerjagcW6lctekeN2WruVQqbPyyUCzYg7/D8D/I6LnAGQAvFuzInYT0c0AngeQA/ChEyVTqZ7oFdLSszGfmEOt1FQhXcV8pwvJBjv9GqVgLlT7LsYcSq/rJZu6cPEpnQs6voz5HJ116lLaKKunGuHQiGr6etGIjLOlRlOEA2MsA+DPbV77PIDPL+6I6kO95sByk3QtFdK1urnq3T5j0dxKTWgOVwtWLpQf/+X5dTm2wDyJ10s4NKq3Ui1FlEsR5VZSzIt6pbLKFAPS1cccaqWWt9VyjkZnC9RbuxWX2OOsj+tgMTTgUsuhPi2s9aLHuhytlL+7bGPJNr0n1RJ2KynLQVETp3S14N0XrsG7XrJ2QccpJwCqW+yncTGHYkC68vEapHRanKe+Jwp6XIilc3VwK4kK6cZfCHPM4TypN9SCjtug9hkAcOS6qy2321WWLyUaUeW+1FDCoY44HIR/u6byuryVsJqka4k5zDfLo5pJVriqqtn3Vaf34p49o/jEVafOazy1snlFaSvr+eBzOxFL5+rnVlpky+H2j1xctzTLxci0smMpu5WWsuCqF0o4LEHKPRPVPC+L8VBV4+f3e5z49jvObfhYAOC2v70Y/W2Buhwr6HViIrZwv/JiukfkSbye+feLZf1ZsZSFw3JACYclSDnroBqNfTFWxWpUoHK+nG6xxsF8CWjdLhdcBCdSMhfBcmiUht/M77nZLbGXO+rqL0GsHkg9IF3F+xdD41rCWYYLJuDhgegFxxy0b60RbcrNzLfwsdrjLiZCqDbTpaVQwmFJUu4Br6VCupEsNcuhnvjc/LGol+XQqFRbGRHXqPf30oz5WQjVpZyttBxQbqUTBD0gXcvazQ1MIl3IJPQvV29BV6g+efiNQAT0F245cBZDjjaqtXajurJWg3IrNRclHE4wlozlsIDn9r0vranH4qIjhOuJZB05GzTm/7+9uw+2o67vOP7+3OSGYEIIENQUCEFAESiGJCIPDkalTshQUjU62PIUH1JbIWqHUVs7FrTVGUY6FTtDyiDGUAFB1AkRBh8gBYGgEUkQkZoiHVFGHtqGplY6Id/+sb+TrHfPPffc3Hv24ZzPa+bM3ad79vs7e85+d3+/3d9W8Rm4WqkenBwaooyb4MajSTvO8Wp9fi/umtiZ15pzF3L13Y9P2t3KnUztVZtDFckh/S2jIX+81py78HceBNXPnBxqqNPPsZufahnJoYqdRlmmTlJyWHT4gfzTeZNzM9pYenXmUOmlrDVsc1h6/NyqQyhN/VKztW0pGN9NcD5zmIjW57dz166KI+ler65WqnI7u1qpWk4ODdNVtxUldHmnPv7mLDhsNpA9JrQpenXmUM2lrNnhkVNDtVyt1DBVXj2S14/VSn955jE8u+MF3vP6IzjlyIMa9aSvXrU5VHnw3odfsUZxcmiYuvxg+rFa6U/fcOTu4SYlBuhlm0N6vxIfxFuLZ/6aq5Wapi475RpeSDLQ9iSH3rxvqXZnh3p81weVf+INM57f6mQ907p9HP7h1kmnR8tOxO7vW4mb+yNLX8U+U4ca1ebTj1ytVEOdfod12Sn3Y5tDo7UeytMHVystPX4uj/3t4FwyWlc+c2iaEn6rnfYHl519HNOmDNWm7cMy43nOxnjsfjs3BAwcnzk0TK+P5L730Tey7/Doj8e84NT5XDDBJ93Z5Nt/32EAVrd57OZE+F6DweXk0DDd/FYPmJHtKI566cxxv/+hk/TAHCvX9OEpoz52cyJ2H4w4RwwcJ4eG6ebM4bjf25/r3/s6Fs0/oISIrJ+5WmlwOTk0TLcHcKceNaencdhg8IUHg8sN0g1TlzukbTDU5eo4K5+TQw11OoN3+6CVyclhcDk5NMyQs4OVqJ87WLTOvOlrqPNNcKWFYeY2hwHm5NA4/rFaeVytNLgqSQ6SFkjaJOkhSZslnZSmS9KVkrZJ2ippYRXxVe30Vx4MwPITDynM85mDlcm5YXBVdSnr5cBlEXG7pGVpfAlwJnB0er0OuCr9HSivOHjmqDc0+UjOyuSv2+CqqlopgFlpeH/gV2l4ObAuMpuA2ZLcA1eOk4OVaXhoiCHBX5/16qpDsZJVdebwIeAOSZ8lS1CnpumHAL/ILfdkmvbUyDeQtApYBTBv3ryeBlsnzg1WpqEh8fhnJr9bDqu/niUHSd8BXt5m1seBNwMfjohbJL0T+AJwxnjePyKuBq4GWLx4cd/f3L+7ixsnBzMrQc+SQ0SMurOXtA74YBq9GbgmDf8SOCy36KFp2sBrdcnsaiUzK0NVbQ6/At6Qht8E/CwNrwfOT1ctnQxsj4hCldIgc3IwszJU1ebwPuBzkqYCvyW1HQC3AcuAbcBvgJXVhFdfzg1mVoZKkkNEfA9Y1GZ6AB8oP6L6ayUF3+dgZmXwHdIN415ZzawMTg4N4QZpMyuTk0PDODWYWRmcHBpiT5uD04OZ9Z6TQ8M4N5hZGZwcGsYP+zGzMjg5NMSeBulq4zCzweDk0DByk7SZlcDJoSHc1mBmZXJyMDOzAieHhoi+75TczOrEycHMzAqcHBrCbQ5mViYnBzMzK3ByaIiXTMt6V/cZhJmVoaqH/dg4rV35WjZsfYqX7rdP1aGY2QBwcmiIww+awQfeeFTVYZjZgHC1kpmZFTg5mJlZgZODmZkVODmYmVmBk4OZmRU4OZiZWYGTg5mZFTg5mJlZgaIP+oKW9Azw73v573OAZycxnCo0vQxNjx9chjpoevxQfhkOj4iD283oi+QwEZI2R8TiquOYiKaXoenxg8tQB02PH+pVBlcrmZlZgZODmZkVODnA1VUHMAmaXoamxw8uQx00PX6oURkGvs3BzMyKfOZgZmYFTg5mZlYwMMlB0lJJj0naJuljbebvI+kraf4DkuaXH2VnXZThQknPSHoovd5bRZyjkXStpKcl/XiU+ZJ0ZSrfVkkLy46xky7iXyJpe+7z/0TZMXYi6TBJd0n6iaRHJH2wzTJ13wbdlKG220HSdEnfl7QlxX9Zm2XqsS+KiL5/AVOAfwNeAUwDtgDHjljmz4E1afgc4CtVx70XZbgQ+MeqY+1QhtOBhcCPR5m/DLgdEHAy8EDVMY8z/iXAhqrj7BD/XGBhGt4P+Nc236G6b4NuylDb7ZA+15lpeBh4ADh5xDK12BcNypnDScC2iHg8Iv4PuBFYPmKZ5cCX0vBXgTdLUokxjqWbMtRaRNwN/EeHRZYD6yKzCZgtaW450Y2ti/hrLSKeiogH0/B/A48Ch4xYrO7boJsy1Fb6XHek0eH0GnlVUC32RYOSHA4BfpEbf5LiF2r3MhGxE9gOHFRKdN3ppgwAb0/VAV+VdFg5oU2abstYZ6ekKoPbJR1XdTCjSVUVJ5IdueY1Zht0KAPUeDtImiLpIeBp4NsRMeo2qHJfNCjJYVDcCsyPiBOAb7Pn6MPK8SBZXzWvAT4PfKPieNqSNBO4BfhQRDxfdTx7Y4wy1Ho7RMSLEbEAOBQ4SdLxVcfUzqAkh18C+aPoQ9O0tstImgrsDzxXSnTdGbMMEfFcRLyQRq8BFpUU22TpZjvVVkQ836oyiIjbgGFJcyoO63dIGibbqX45Ir7WZpHab4OxytCE7QAQEf8F3AUsHTGrFvuiQUkOPwCOlnSEpGlkjTzrRyyzHrggDa8A7ozUIlQTY5ZhRN3w2WT1sU2yHjg/XTFzMrA9Ip6qOqhuSXp5q25Y0klkv6/aHGCk2L4APBoRfz/KYrXeBt2Uoc7bQdLBkman4X2BPwB+OmKxWuyLppa9wipExE5JFwF3kF31c21EPCLpk8DmiFhP9oW7TtI2skbHc6qLuKjLMqyWdDawk6wMF1YWcBuSbiC7kmSOpCeBvyFrkCMi1gC3kV0tsw34DbCymkjb6yL+FcCfSdoJ/C9wTs0OME4DzgMeTnXeAH8FzINmbAO6K0Odt8Nc4EuSppAlrZsiYkMd90XuPsPMzAoGpVrJzMzGwcnBzMwKnBzMzKzAycHMzAqcHMzMrMDJwWpBUki6Ijd+iaRLS45ho6TFafi21vXoE3i/JZI2jDI932vodyayHrNecHKwungBeNve3sma7iSdNBGxLN3B2iv3RMSC9DojP2Oyy2K2N5wcrC52kj0/98MjZ0iaL+nO1KHgdyXNS9PXSloj6QHg8jR+laRNkh5PR+jXSnpU0trc+10lafNo/emnZZ6QNEfS+3NH+D+XdFea/xZJ90t6UNLNqa+f1jM3firpQeBt3RZe2bM41ku6E/iupBkp9u9L+pGk5Wm5fSXdmMr0dWX9/bfOdnbk3m9Fq8zprtxbJP0gvU5L0y9N69iYPq/Vuf8/P33eWyRdJ2m/VP7hNH9Wftz6UBX9hPvl18gXsAOYBTxB1pfMJcClad6twAVp+N3AN9LwWmADMCU3fiNZn/nLgeeB3yc7CPohsCAtd2D6OwXYCJyQxjcCi9PwE8CcXHzDwD3AHwJzgLuBGWneR4FPANPJetM8OsVwE22eK0B2l/V24KH0+jjZ3exP5mL7NHBuGp5N9tyCGcBfkN0dD3ACWVJtxbwjt44VwNo0fD3w+jQ8j6zrCYBLgfuAfVKZnkvlPC6tb86Iz+uLwB+l4VXAFVV/b/zq3cunr1YbEfG8pHXAarJuD1pOYc9R+HXA5bl5N0fEi7nxWyMiJD0M/DoiHgaQ9Agwn2xn/E5Jq8i6j5kLHAtsHSO8z5H1cXOrpLPS/9ybuvCZBtwPHAP8PCJ+ltb5z2Q70XbuiYizWiOSLiTrvrn1vIi3AGdLuiSNTyfbsZ8OXAkQEVsljRU3wBnAsdrzSIBZrTMd4JuRddb4gqSngZcBbyL7XJ9N62nFdA3wEbJeTlcC7+ti3dZQTg5WN/9A1uXyF7tc/n9GjLd6pd2VG26NT5V0BNlZyWsj4j9T1cv0TitIO+7DgYtak8h25O8asdyCLmMeTb4sAt4eEY+NWEen/8/3hZMv0xDZ08Z+2+a98p/Ri3TYJ0TEvamKbwnZ2Vrbx6Vaf3Cbg9VKOkq9CXhPbvJ97Ol87E/Iqnf21iyynfB2SS8Dzuy0sKRFZMnk3IjYlSZvAk6TdFRaZoakV5L1rjlf0pFpuXcV3rB7dwAXS7t7Fz0xTb8b+OM07XiyqqWWX0t6taQh4K256d8CLs6VaawkdifwDkkHpeUPzM1bR1ZN1W3ytoZycrA6uoKsDrzlYmBlqkI5Dyg8VL5bEbEF+BHZjvx64N4x/uUi4EDgrtQofU1EPEPWRnBDiul+4Jh0ZL4K+GZqkH56b+MEPkVW/781VYl9Kk2/Cpgp6VHgk2RtKS0fI2uDuQ/Id7O9GlicGph/Ary/04oj4hHg74B/kbQFyHeN/WXgAOCGvS2YNYN7ZTVrMEkbgUsiYnNJ61sBLI+I88pYn1XHbQ5m1hVJnyerhltWdSzWez5zMDOzArc5mJlZgZODmZkVODmYmVmBk4OZmRU4OZiZWcH/A09HDwZiAAAAA0lEQVQgq1zuqXC/AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "err = optimfuncQMF(b)\n", + "print(err)\n", + "\n", + "#Restore symmetric upper half of window:\n", + "h = np.concatenate((xmin, np.flipud(xmin)))\n", + "plt.plot(h)\n", + "plt.title('Resulting PQMF Window Function')\n", + "plt.xlabel('Sample')\n", + "plt.ylabel('Value')\n", + "plt.show()\n", + "\n", + "f, H = sig.freqz(h)\n", + "plt.plot(f, 20*np.log10(np.abs(H)))\n", + "plt.title('Resulting PQMF Magnitude Response')\n", + "plt.xlabel('Normalized Frequency')\n", + "plt.ylabel('dB')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "Collapsed": "false" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.4 64-bit ('base': conda)", + "language": "python", + "name": "python37464bitbaseconda58faf23c4b5f4fef93406f29a1005f35" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/vocoder/notebooks/Untitled1.ipynb b/vocoder/notebooks/Untitled1.ipynb new file mode 100644 index 00000000..7fec5150 --- /dev/null +++ b/vocoder/notebooks/Untitled1.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/vocoder/pqmf_output.wav b/vocoder/pqmf_output.wav new file mode 100644 index 0000000000000000000000000000000000000000..8a77747b00198a4adfd6c398998517df5b4bdb8d GIT binary patch literal 83812 zcmXVX1(*~^*LB;t&+N?3x*H2D?ywNtB}j0$0KqLlg1fuBdvHi_ch_Zo*2det`|tOA z{;j8{Yo~jvZha`i_ zF9ATMBm~d^dcXph05jkM>_7w%4HnYA%CjMWl|!;uE>tYc zm%l5PevscO>81Qu$-R}|%U_kZs8H_lDdnItBa4dtw*RxhtXQE}EKOEytEn7WK#F2- zhLTGFVwDuBB)f8t8Gx0aO8&e2Q~4}Vu2w1canDORzeI?xDc3giOqfnR_wKqn>F zK{=WL4S^a;N>!{&RqTpTYEM*JFQ~kGh4PFa@*DY%d`aFTZIJuwPN^T-& z${BL9oFu2nv2ucv&yjm8*Z0T|m9|BI#=vagIba0`fjhz1pbn}7^@G+yH=*ay1LzdA z8X5&PgLKfp;22O3o&Y)n0eQXLLiR~pr8bfzJ`?wf^Ti=z2eGD@CdP@7SSq{`$^@s_ zPV6T(5o1M8FpJ~F0Nd8^c$UCJp z>5^itOR-)Q^kQRithiNtCEBFX(sQY$d|R#y`~#$bQ@~4LDd>cHL93wK&^zcpv=Qn6 zQQ&289@rApfwzIdKuDe^$H4kpm<2olL?9XL1darkfJc-kyaHc}CJ{$W!_RDEC^uorlt{PGa+t1`B}DOZz$)<9k0 znLI%00|vQ~JYJq5uarl~_2q2YrW~!65*I7q=E(1)SyD4eE14Dhvn5zMBihA&f?b#) zBuWS6KY_MDd)XpClgEMU;WH{iou-+uo~UZ4l2l_gvi5h~AKLlqBgi;d3(L?Xhyyub ztSpN+1dT9nfe{-j}RoqwZ3s=E~xitPO4~Yk*1xio$$Q_gs`w7Sc>%*PkJZKO! z3?zUoaGBCOyMx()Q_hkIp}w$;f6Omb$|+<2We%`r<~!4bxkHbmtaO6-R`$ayfaXG$ zv{3GgBK^hOxv&09ovxjz59{mebF>SPFjxx|fL)*( zbV&=iRqPoWqIBdC{4UW0KZw6X*M&!6d0{(xDbSbA01l~!s*9<+bPsu_`iymh3v)-r z_KWJqP_z#gi}z68`7cT! z7ZPm=m|VzQmixgas@tk}%2;&5Tj9a#0s4CSt~#^&I@lF_DA!ZG%XmpEOknr0ZD|jE ziMWR`co2I;_^}6Aw?IAaHP{XQsXk2b-1apl5I52t zYe`WLmLdQ*G9EswG9!=WgIo#LG_)4GPJN-eQnQGG#Ab zyf9Wc%=ZV~>Qm}>$Yij$($mg}r^Rs~3}2S>C7*DZ(lM9Fmdr91VEToYdaE;$;xhep z^-|o8|3wya-L%bZJ)^FsSYtoxYr{*x-DXvSGp#uJa@00cq2e>XLLXIq4c+w*wVlCL z0jz2Rh(QV z&JgZOIIt3crAq!V)s?7$cSbLg!>CTgqd@Bb6gYrum>c2g=o{GpH_*6M&8aMPyknJ9 z8yAdS9*?K~5?^IM9C0vmcC0^TTimROd&Zf_Ik_uzUV6?#yq)bw^$4d{6qjaI?eh-s z=KDVSzM^}nwwMZQNp@s%pvmfhx|?yLse$>6X}msGU)|i=l4l-o2&uY3O@Z;kD0Uzn zO-&;^69~G||I`aqkySf<&#O`@-dEN0^s36P{OX_ScS($48Zg4R2s=l!_CArnRXdu~ zJ3Fi9)zoc{HTH(_J=1&ColxUo>}7kT=1=jf2!cJOQXo>SLu3T{mOd|7TY9-{VVR*~ zM5Qx0o#4>tf&4%RY#7&BwMTczlx|Hi_cD}f2B@~`E3D0J7tH(Ax!^YGJD*QqrMi>% zDGzZDvj?|&7kGzO=2xmJb3JX!PkD-}%z?WVU6HUbAF6_zRvDlR)=SZyYi!TnoINL} zL&_|t9DO4hs=21o-5Ns@lAOQkV<4Nl1kP0Y$QiyAe;i2hj4nP{_Ptd0+^#69AcA$# zSpNubJKwT!p0F1_jWpIgQN^m>A*rAmUaxIsoM>pIE|hX5WiA!GY<=oBHIgVHmWFEu z=T|MQTI%UsR#2Ac{k19!4MDS+*6>kcSKzW{oB3m0VdkccHaQ=wUyM8BS{PrI-k|Q} zdbt_S1U%AXyri0;W}r`SEIgDOfi?ACD?L{Fyxdg+c|LmdWj%uh!A1T*0&$_0cp^VV zx&unu>-y=2`Pvc6Jh@qYS9475M7n^#gO%c5W;jlvOVHLtukftEjHZ!GB&Zz6Fr%7F5PjeNgq*wPe>Rs2AaU)5N*r7S2 z9*n5r-s($04xNgw@upS|_hx!J6+SA8DT@qkC70lB!s_sgP%7?W&P&_1HOTUiUVwxKO4q+F#-rn{aw|?$&M)p?eztONg&CT`*~Hu8+2BpChN-sga>kgPlp0qXe8}*| z0x7>|#MWBV6sUJSD?J^Ks%_JoVzpW|jtrGDsZ2~0i1SXWI$HWN|8C)hqOTRTL&L*8 zf}=t=1E)iSh#pi^xs!U8Y96c=k1(r*58^$+%P-=tunBxRRRg^gTpWz>&GX@&MWy>o zHbR8={~CFHXtcf`*yvI24b~>XkA>JwCmY?gZB+fau(OnC60F8 zGH2^MXvQONz_~0#O~<~6&A~|(a>2g5wEWhdpV$K+?XoebUJjh&@32!9Z=~X&jHSA;gXlP}0k@i&$osiY>|A;h zwVBz-MX_D!-)WQ@O{(!bm_%3S&Tu`MJ4{PHM<|u%K?Ai#hJmJUMz{5o?YP6~+~}1Sx}&No%mQ_!9YP204Z~1_$?aro{42U3ToxD`JP}+J810`|RaFj` z**r|izS1hs`!b}WN%>!8+seFU?(&o6@5+Z%4)F;=7j}jW(Z{*2;#%1Tty2GOIA>k! z>Jw#)y&OL_se4LNdcX7s8Rs%1v%Y6sO)p73lDs16ZCv~4Qs-t{j(M5!srH*HAHENs zl81-_yNuRTsbnUx4h@DL2f<)UK<97ZbyfAR+*w|;ETWVweo#`w^TRW+EZ}+N340!S z#(QpfUX`7$nCCs|UlY2GRgl-1{k%~Mfki4=KgHsRsOjz>lNX^bLo>`C_&cy5K9=7Ox6*^9HkSC6iKAbVZbwDjgF_2ZG~ zJx;{_+PcE*(eKslg8u|K@g(1c?LyBZlZfV+Ba|HQ`nveW`mR*jD+YN=i}|AeiqOKJ zg})b#EnZL(EXgYUQJPq0D1Tgb((~2>S2%p*LlbZpJ&QTdH5Qu5pCGk%vMJx*I_gV| zFK$CZ&7|F_Ovcfy`_Q-BoF)r=bl+N+3qqn)d_PW-crhxvA?v~~UoG14d z8i*2D6uz5ndoCn3J(sjwE77lk*dhDrqGI3_|=`>5`;jFIN1GAT9 zcggyknVMEB<#OWn*pX3 zwJKMY{a!k&_&{M%LH)vJg&9R;QA*Loq8ml~ilvfpX}Kq*yhp{Ls@MLRp>w!I6>;~) zC%}C~tp!Yvt+iZ-qS@H~3Dc6!r6gxO%B;+ivcT-*Y6ml$WXww)n=&FHFS^j>b{w(X zGqpF|(q?LsRehk7zyz_9oxrT7G}Joc2)ZWpFtE`FS9K_VUV5UqRnf17aKV6l{O6kd zkp-RtO;P`%1I48!inlC_E03%k=Bpo^g+v<{lA%Fw4--bJ8MDY5I~(-I4k z?xp5ubg8DQ{wBLt_RP$YnPbzorj#e1jZKa&cK%^sZuOWC8_sF7)LY;;K)Elu;S`f&unLi_cLBZvMUkgtZo+;9ms7s%h&hq>y z+fu3bCx#yo&*^>qO|d4RL*{7IrmeOik$=aGjW;FvlaHq6WX#DtozWA?-> zAfWqcHhU!_T}$N*A^p1l?9J~_Wt=KZ%y9)yyia> z^Vx#cMSF_pmcH_2m31wDT=B>YhY)-XJ%Xz*4F;YgJ+#-2qir7}x<|#u4v2r8I4HR^ z^>W6A%>1kgS!G#ZW^Bfg)Q!n?6T>lQ-RqsF9c}H=)=?%ze^dPrycsB$yNZdzJFX}D znO;sj2$uvjevP+IrMrB4DP8=r=xRZ1{=*;XKe!)P^Q`$l3)U80E3RJpzI3kVL7A^2 z-uG*0H8zFr&V_}bJQ3ce`KiBVNpk$@?iBkZ{&-?`k~(=<+P(}vvqRRQtXf&QneOy0 zNmCP;nD6c$u4j%8_Pe$R=7omK+W%D5pk+X%bVckY=5v$TFx3cu5t<%I^0~d?%IoC= zJ&jA)!ifb`-i{y7@^1fJpPyRrXW{B1Yf0l$b6MRAv2wHTOi)DIlY80sJS;~-%T()i z=|;i=Ip(-FyN^Zhi2W_TMIxP4p6p41QV*rpNxhabDMd_V<2Od9Mi|nsjtKBzyf)%R4V={Y~i=D&nN>vD*P;<_qX=CE2oz?FKb^46*Ed(t^RraXHov~ z!WqS9O42>{vZdvCWwG~w-xK_d{z(eVSH7<_3@m_`s88wM7-B6uY%3#5T}L7(yLUz9 z##rM%#{Cn&BVkWs`$Sv9!MIB?2cus`c6W}oH?`_4P0V#o9SvdaQuQ0;06Y(B25tiw z`JQx7nJe$pxfF)KM^}d|fkN+^%B=ESPw!Gy$)n=W#kY%_mb@v!N&{uBEAlGuc~|)p zgJ(k<(Urso8sP2-(Q*%HG4fhHTRTc0HZC;pw7#*eapX8X&aJKyk+a?1qpn6Rh#nH1 z8%?>ly52>sw5QoVS=O63nnoCZ=mXmRn%e6Asyw6yl8#iterOJuBje&Rp*uf`)zEK= zZdf4HBrw^xxN2bKzZE$Zx5`hHC6^U>T9u70e^7D1YQ9ew_#9jmlEY^(gltPUV_OUD z7;R^ z!D2A#$7!2szNp-4zbamxrgo}7s9LD9RSV#w&?Im^kS>RXF_F}5^aGU|*;dapJ?dtU=<)moLdtu|eIN7GHy zRC7*qNZVUCTc_1!Xx*9$)j|Y>dx1UW3qmXIAl;JOjE~07pqtRvs1VLTbJ6o?3HmEG z1D{M@q^{E`Ohe`f)0zF7ozA{v`Y@}RAd|=0;o5!W{5BU+oFp-yz$-e;wz(H^V zoT0j`j?rf5Abn@O#W38EWwe^irk=Nd7~$t3(^xG-4dkMNaOt*V+=wZn_}9{XzhSNrY3;$W}v z_3(BDC0Ijw=o)M~S1P=degeZ4j4%Q|0*_FQP_0(C&^*%i)t%Jc(GJnp(RR@3^&Wk7 zL!llpV}QbLp;VXKITzV>HXveN_n3 z8k!Hhl!C%G{vn&nOrX@{Vf;QiD7+`wDL6P-JM=o-AM1hd!p*n?AB+zqZje68#Jp25 z5k0qoOXNWzB>2TOl0(6nlK}#F0$ze@!v>@Z*{ABRzOA08nW#1C=ICbV*61$kg1R>P zUiu+=QrA>#lF9)ZbJcrd_S=p!Bz} zCZO~@PGe9GSp#YR(;U`}*61`F)OS>+2o8H72`m8a%KfExLN5P;UB*t}!GG`vx`DGUnffY6Gs&UI%OF+b?1^dEGlGchj!YqERf&m}nZTutd zJ6Fy<aPkQhmhsSD5O%UuK0AD51Z0b#xHoxjKZ%hqH2v2)mD_HQPUSZ$M!1yBe0Jv<*7q^hOfrrx7ot-h|Vr#Y@UrD>_Bb=L$<%_ryt`qO! zdkI5@;lethSjZ7uiJ2lI0t!bGFQ$m&MNqmUbx>G`f8_{;Y3K&>;0owBxIZ#gHB#Ym zqBMOpD>Ww-YmRFQHC?rbwR5%4HIQbGy0KcN-mS8$79$p9HT)8)1T(0ZxfHI2(m(HT_4va| z?`$XB5L$>2#2AIW$d~#mY~VZ~3H$+`h5EtYU>g#TG)DeFJ|f*zcU6(<{_45vp=wsO zLN!RWRP{pTS3OhO^aIirc?kD{Dd;iuUTOIO&lJgy;WGZQYap73D<^ez&+qwa0W6MnUAbPb|Slx z1IRpN8uBO78A(J6;6GsD)Xe zM)=M~b5}S*!8fjPYJMU=UN|aD7XA@hi*2QA5}@!%ZRBYR4>=MT4_pCsz)%p<4KZYE z)euNB^_N@17maDDX>)mp>@r9jsdW|W5pDRXxBO{RUeh_u2j&)YnE9QltBjBN+-Ie= z_wz|Yp2Fs-q#Q{nT^FlKh0;7A73u`vhR?&pU=LY<)hYREA-^qX~~byj_) z{ukqXqut=t{m?wrI`rlG-g=K#!GINe1}pVB4W0!X;pSjX+8A5uRoZ+N4gUq@LaPyv zYO?ALq>(mr&zSK{nA;~U16lxM#CY~Qb(jnh>C{jfr?!!8@MCBrTp%=5H>w{g;;XTT z=qzjsv7CBOnaNaq6?z(NLS5jlNY7-GxRK8gmP(_Qn7_i*&T+^&*)hR3)cVqjI!M=B z_sFQ+$WD%st)G3Z{jjZ#<&NQ}wzmpbyly0f!po35U^j6pQ-LE`3Sne=a{IZlOd8dM z7=zsid&4}o2e;un&{pC3=npKNh#(7y$@oEZdU!J05iiA`U|rGK;cfU4ri1WG;TPYE zbws~FN-@Y|!`+B2G0WqQMR$nY;5Z#|(%n6-dD4wUU3^;1h^Vy4a}kGazgS{SQ+4f- zD23%KRJ>n%31ePhaKPmq;VTW*$3NiBFinW^ZL9iFF`)9AFFlkQUJ}X*CI#DvdWHW` zu)eqG9P|}x#5!Zu(C*=*sKi`@dMRt3Y?u^Ri^ssxTHJOq_EhTh%r7Ysu`67lvoQKc zvbI_{+nCuoabJ|mwZeYWQq9;}|5jU}Y7Aoh64paC!#f53DgRmWy2M$2)w@3!AJ|@b zp>%fv)|XW%#Szp$m44qrz!k@Gcoj4O=$ zR5~yh_UJm=d&QeF^femf{FlByen(W@m`7=HPM>-!YiDF#i9hb{5rNp!E&WaJ4ZXBY zp(U&in-*y0EiAuKa;mV0+ zVrDz%7&Cxs#DqYO|A)VBI1~UYca;_v!uid9H6wc%S)xRYSfY(^EGv1%E-ii zG3kyckwX*qW=yKX*BX`CC276e8nMIOEc#^h*T@5wN!rm+SD`IEh1eQi8`|XGS6RO- zx^z;>mBI%F(|&>lGfOX3boS;~t**FKeywV%zbCqd7)&0e`_msOl>AJ!p!@Jgg<|$T zm5NWMd+{+UT6S17Gd+cyCf~lIFsp3i%;Aa6Y0EX-|C2ppq*VDZ_O`MmxNs;j?KiJ=;yFC z==V)4`&gLv^Us1Yo)6_j<-v+w6;LHpneY8Obf0)iXEE{Q2cnQV!!43l%aqhgTqgAA zU$ZONHS`qrskjuJrGqsi|ge$|Cx7Nh0~Ml$*Tx>Klz8Fn~6SD3H~&Qh8_?I|6JN9{NS#0 zOPO6vcP2nzrz?1a@Sq_p9~JhGgyPL zgB?KrBo2{J$Uo>-ObVMx_h%pRL0v8FVDS%{$5K^W9ev!B?Y*4~BI`JZMD()!9ecUj zhML#XH^s{ChtV$+y2W&i$%#E2B}TkA?N|F0U5Za=ChZcx(jnC0uU_u+tS)&_`oMcF z@XR~P)3ESWVbAi%-ob&bVF2rZx`I=y#`w<>yZHnG;RcZ-h$Yk@YAw}_?aWnUesU-H zQr$_-7;y=M;j-$hBiik-k5`bWzay?hoV29GHLTX5`mbq&VrzP&7=yq!-U_*5Ax!3)@eL*?&$8-mNj^TNi^ zSl`&njlS_@Jt0|K%JTS1Vk~)&>dn}=mf{@tSN1WTsJ*SbB7f%9_(XPO5(M%y3Qdg>AT(1d%bw-W0|H;Ighycbj9>f;(7wbi}X)!bH~HzTZqSr&pb zfxpDNR4S$q?exB_{I~qDKRGXQS6T3HNL~BorU}ROPrd_eQDNzY;T+ud%{X*nQ$6iOQ`($Jbm)mke za|ndMUUCy~9ng#)#>~LzKrP>i%Ae&c`~%T5p()eZ>Y*JvY?;GY9t6+`<|l{6P99&Ih_d$x;g0h z1l_Ob1ZMjxebs{}(S80O<(rFkmQ43opc%Lp{Wq+|enIa9ZiY59qrjVRb1{#$a)Tuh zc%>jQmB3l3yKJXudY9!l3D47(Mf)uu&3ha#-FsZaqIA*sqFcL~ z>piM%NP=pKsxR^hh!vGZN@!znk?*KKI(Re8pxga96bJ7h+39rV1uhNIUl%NKgj%DKhU_r zkg6(#`Wo*>wN5Kb>gp=DoHws^wTZkNH7d47bZO*e(^b_;_!<(a`CGFR86kCJj-d_1 zw%~-IC%gr1LY9X1Ry{BOSy3%a5M9X_YNirHUdK;{?_sYw63B!YA%_{tEaol<7(bf7 zB<&Hq@xAfR;SR(h`JJ{o5X*V}c%XrF3tne{%xm2AI%3GBe>WmE{JXCx7CRK#j4@1}u_LJ06 zcJeN|fd0zD9LyCn{S*``hrNh>2+bsZ0K2vKfr0$M;Gxj(QY+-TAz*24o@mV1R)=P( z+Bh2}wofgHO}5`Lx3e-4S*}LWdt<}yE%u z(Id{qj7)WDyF5YhK~eO0HeS(okKq~q6MvYSNxnm)h!uQ+)}>l5&cN6B*02wdAG$_X zpDEK&SLcD2f^|$0F|(5J_y&$7V{_wwmg|mxT$A=&B+{87q}8Qgl{E(rC%`D6xGap{Cs#e3NbUqkd)4!p&}_4)kwil zy0A<6g~B_ELI+^uxDjd}d_~-j_6hA4LDhZ3ICD*d-I$}(sD7%ZTK|RVXzf$YOQ@+lh8szd#1_iA-2!(D9iT z)LJC0;Z0Xe)wDl!xvC`KSAD(6dhv_n?>iP5svB}GCnKu6PDbu`<=KBR88ijRaYV2A zsr#<_1)L|9&~{oNpWwI1+Dtz-fy_iLf!_X?*zeRy`XX(iG=vd*jQ)!rq}~We#JXHh zax8I{JgTftXR)u?>ztD=P`G*3TN zqAhcv#7|;NNF&*WnoJy3*r6j>2>(fM6Z(iBS&A$rAJL74CHy7s8T*bE>A841VPpSL zT}7Odj+pLc>3@*Ex&zjK^b6Gk)i@|a7fe^9_9V`ZpW*0fxUZXPy%*Umwj_3R)MH16 zak_fHYQ4I>p_QpbJq(QC?MnS8kk^PCluRF`75#DWi1(>48ubw}g_EE0b!g9UM))MQ zjIJrxm#Wz5#1ZTZaf*Gf>>#jn8`weAQEUnp%N|0?p^M@`d_y3a@2DD~uVLAvzpSZL z$EYC9?>0K-zk~y^n;ntHF2=czgVAH+_rzwp&RSaP&nS#gJ#7ucK7A?@EkO)UoTPNh zIC@RKCIfiyaQ8q{Utxg6{v!L3&+&6;D%uSlht(q+u|E~0eLwvI*W%NuLELM;mB6w& ztbC<0$Z0p{>NctqHG9?b^*?npkzLX=@*Qp=SCI?Jy+kI35+!K2KppR$fEQa!wWVGw zsMU6KC0Z1IgYRSK3I?HoYJ(nY$p{AGSn0Bnz4%?;9 zlo$OCkEdQ!$s|r}#?bJe!PegQ{xCY7s>Up(X5eqp*=SGn8Znv8l?KY4xDHBOdX5OP zhyaT5LTC0OQ6Kw)HsDR_cW|NHhFB2n#?^+0>WA4Tn_KCfng;MbbvwuBxNV7^*qRY< zOlwTP*<#$!O-nh&2`Ng?Rbp^A*CtQSbQRWjwb2f2!wo${e>x_0N=d8 zDr^<4W=;`0EE3hCbJ0wE6J02*7OFADSPHrgzets_Ab(InQHJBGn4S9sZ--mUt*PEv z56Pq%X~?s6vh+3X*R?>Z>yll6C$>mQjs45M$@Iu{-X0b8TU=(`pUxb^2sjE-tJ`S0 zYpbhU0#o=C#Defy)JXQHN(e3SXLwo285-a>hT7w=$uYE-{D?osop_-#x87slixmn^ zB2!(-PgG|%lShSJ3coXw44~cdhx}2M4%SH&dOL(kr!~(k)g0B#mBzX{oobY>yX%;u zWa%7t!8%Z%XEfWR-QS|WyG{1p+JQi^G!E3mHB@JTQ(SL)L-#Xh&|t ztK*4ifM~&P=hsTr#8!M~?kV+%oWllOImj$F>mdl=^(gHDcdWd%~0a;{v;x zb1)+h;xYsM{c7P~mD7@En`drks;A3U57A$7k4tl8MkSKA>dG4Skz-Ai5L4hjVLPLp zDAU4WWhLAeJ|h3hHX(xBGWQ)id)Bx%sW0Z!;SHZ=~ZlUqgR=%Oa z3+|WRD?4-6^9+e%Bk_miF!DI{iR?-?Ky|*izQWJ|@tR!3y}-n(@x*RIo>aaB@VM6-J)KgK_%p(l{npyk1sNZAiI_sElupif+w}~e3C{!trWNTv1z*heN z{2BFui{N+BtLV$*9Ws%F#JNCYXbx~+ZYLqaS}uuuC!_$)fFIS zOr8$E^bhpa!A;7Zs!RNZ@XK%?zBe2)4YVz`owl_ygf+_y-(7c-Yi2x8bh*k*^Yuov z#v!_gMh>!J>NyIsQN?=*{$I#mBUf+k&#xSlX&Ei(Q3o{1(1K_*?RF1L;rL1iT-y zljuhDMIWM3p%tNfVHa~pn9Q}OhlJLX2-sgY&auw^(0b0)LC5JkItmk_(%Yu7(LXGa z#&gCM_Bi(q_aBbUx{-1l1-mNXkIM&uVca6(NnmVXKI$fUCWrn&eWY4YY3xFF2)|u4 zNIIF8&PkiamwZR=D?6J1qQu6jir<-qA3-}|`FH?tPD~7s2~7*ur6AtTePkw~@k~=_ zv382}jAguakad;uxxr(bAKxQ=LAoxc*g_cAnkG4NBlFzT98>g(@;%C}a6LtU7udjM z5a)dV`38lnkv?`6Q=QsFs>y*gs%RzODSH8rNEa1kr-d|K;)Np|%kjKhpx8%bKjH`e z1Gf;2;tRWD9kDv0bfSo_&&?Cs;~d!(+Mw=krK~C|Y#(Jp_3bV7qfIFj(vHWaICh#o z8T%^huiKGlBFc?numSC$W7w@?AE1(dNio4!eqCrDPSQP?xy)KxL!YF(F@yP<(l@!i zTtfoHHNqz0Dc3_$!Z-zsKr*+;iNq+}KpY}eWPP#^-WVGaK0@bnEx8YD4XhV830a{X zWNl*R%~jTqhGT~Pwk5IF6npYwceJ&mxwpC4x;>)I72&97NR&akoVmeyr6F=0H=qL0(N=~}Eyh?4$N*uZbXzrqdCuizKW*&zE(Q6|JO4k8JErmVHP z61$1bM0a98`V!s3vg``s7Tq7ukiNs!3_eTHY_toOA%?An_y{6?bjq`Yey%Cj3+DIc zBi0el>8=yD%i6KxcJ>n2Q1~g$5?6EM$?D;jp_Awhaz2yCoM(Pvf^Hslua&*x8(&ZA!HvF5wuq79WFeB)p39JY;g%KiQk?Bi6+|=KG0B(swaN+$UV&Z?Pyj+Q?x zfOW0?Uc?7$ZS5`T4}OPuPr5C)62qK|UWsL6xx@ixJ@*IOj2X+UVY_hwZlZ#Vz7<#= z5GW-+sL$=A_fVr~MPI@I)P7}`8Hj(w|HA!56!}SW;274JUC%#2oAJ+tXEnA{xv-v;UV&{E_*?17zFLsu%NsA@Bv_f1e45min z)$mPJZEiEWmz~Rv=Hj`3nRvD-j|-=TDf}6(8P}JMq)`Pmw$s<><)oWP!FOPh!~vzZ zY$QGt+wiZ{9&Rv;^CKvpiG^ywse0Z(853>&jec#iaaF{KnC5X$+`S?`IAW|_%(+(H zVRCh{PSzY%v?6KXQMpW-CT)^#a1W^6R2p-DYs)U8pEC`)<9r!+idB^V>@fDRqAi=k zZD*I#lrn-v!b(md3i0juPGv0b!RHgVi9g7b_}|1>Hbc<`Yq>rQB@YB1A~$po^aa*7 zmUenn-`i?(HH|*xUhRD0%4!CYHzkFb+(&UR%4=BT3E7|O!R+jgKQlTpNY zqB#kZMq&`2h96h<(isSdyh|=oX7z1kHocAYG55Gawkh{ho(#C*-!*46O$`V1N3>&% z?X0AIs&jV4g@`WB65B27QrklNUydCXk72pqpa26_n@L!!hZ;tJ6V z&%;__xoAFG153is6C=q)N=v__K>8m#fw{}pKYk7 z8b_HO*8gm^9ql9fM%0Pu>{w{OWE*A$tdGoJjDz$wG$qJu_y<%4ZUW~4o8$;dFP!8D z@X<s+4ybC5#s9Nblt?U}JcwvLpDGZiGQFPBnkCbg}hx+>LnYRJ-ark3<}Ew6LSr z-Iku_O~yX@kDA-6Zpd}$Kk$~K`f|uO#Wuo8u0A`9A?a827Wy(ZmGtBHFj9#Ma?ms3 z1L3vdE#b}K6XEt~Kdc6EnY==6pqn!JOkZ{)`-E-DJ>}Z+U-(L)m*iA*YFe-mnyVr; zCjD7Mtf`&3$RgTmMr?2zT@#%bBibm;-7I@6+wYberV2xuuD*7k`Ww<1X2E}y{qAac zt0?hPxq9qMrZw{)okiEDekh|i5`T~N!0w|R&~_+-O~d}ha4Zs!BkW{z>KpxxS*$Rc z!xdfMP4=RqTRP2dP#B`1@Kn4Zy_H9T{o#wMY^_HB+HlTv-cn(^=-3wVi*v1Wp0lO1 zBI1Q(wtbAXw)woVp5c+gaMe|p!W`HNXd?Ggb{1XdGZb{;9NmiggFHeEA~@WFPsARe zm(Vg)h2>)ZV3QQTa+YXF&LNvoQ|W6=4P_1SiF>c8lIC(W_m11G?7+Uq{iE;_toT?# zau0wj;j1c2W7E5h4@_#y9qTjuBggiLy%F^zMnp_-sO(nTUP~MEcO$3&Q3qROZTbRG5z zm(TYX>L}}o4vJQExHw2G6cU8d{A|9Z;8FG#jaJldPr&Z*4%H*gWnHA9w{f27hoYgL zX^XU%+qT;_*+AQP>mo}(b3@Z!L)`x(=_uj|xc0bryE%{3vju%SW&Pj$i9tTNh|lEn`GvxtVx+WI+K1l%^U!#hDJdmZ$`;#6 ze@lOXtu+B$nQycuGlV_N<;&s~C6%$NSa3qCYL06nw1>1iv=g+YwMR8aG*KG0`lYhE zLNC9^wPlyl2|vLHB|A1y2q_eyc#$#0gnOD#wRu-U%T8R*dOfB zXWSu6>0!E=&Z3K`hhCxcXg|7^=0jWI64+}U$X)V*ynwn!KXREoBi*1gupV}IhKjTX z(}am+)J!T<3jEn$^bz&beKdkDg!V>T(iMzFL0Szqph2`j-9af<5mUvn;x1tlG%ynW zr*`2Hzlb};E#ez-9_;BUlt1bcFA+&a@JqLlLu5Z`4%V!PzNcogn7n`vNNv)WoPYwz zJjpJ;7l)xwQ%&|sy`k^W6N)8?WEF02PW+Tas?#Mjm#$_0V0?5etxYfD6K;W8#4UXC zX!2F+23?4SQhjJ!{17?|1AtDh7Q0}DmxRqiM{%miN;}09@QYm}mYjwb$ZRlV58%BY zgZH`+>{l-|H2#y?kpIYZu>R^wHE;(oNy|tClu|aLQ@$T*1I>`fWI9;8CY)8CQ1G7B z%q!Xr7I1~gp%$_q+{?A37`X_`El;jMy`UdXRX^|n)nYg3UsRVKh_l42LIrUn{drCb4fB2Q!)pXR1;Qc|(@qbG?vSN{8{Ta;YlL&sH%Gd9y+M zEQ)v*_r-@`w9dr44gwdoDi~~oaMrTK^Wt;ylh{?fCDezKM^pT(4PYa86;23;#5p*V z1HmuOmX4EZR8OISK#$Q8%rRy*Q-*28?1TbI5wi&vl)?-H_c)yyLcfx7^c|SIQ|LVC z#SDa3SxVkYky1L?q0@l-XA66U2-wXv(TcrG6*%!3c!sM)4=~G2u?cisrim-WMCq|~ zR&t4VVedz=>i6O(@u;{3`aQFyS7LWDRjLO~mvA~0Y)%si($S2O*^l>CvM<<~>>?(I z4rabW(V{9dkq)3I=qK8m_9Ew?9}@>XosLpxkrUS=(K!!3>1wc)lchS+5|I(7h)bb{ za$b5O_7Go+55Uo$Bi$4CVLijKOC>}J3M|{CsnXx#Qo$*NOShy^SY>Z%30Z)5SV2aB zS9+ak!WJ=g@l+NwpTTxri>Gy)dB;p=7SNMqHPJ%tV>KB8izp?XgN=3;PYB5(2Y+W0 z+lbA;JPt=pzAE0pe%+PKc!yuGjG#D4ngC~eEVYtCVa5N! zcGnU$ZI90!K}*wG^fVp7+++l1BU6caPhW$@n!vOL8{7u9m$f+Mjc_(+L)B#>yzX1^ zmUu+e!8*1>#b-56bzMBE#nKVz0^P!|Rixwi7yrOgyFxdmo9Gul!(wI%ccGG_hexY{ zmFI~)q|V@uc9ce-H{uehLf7EaF2%W-Momzxd4biuVJ<^Ar8#qquA|p!38>`6Gb`v( zr~^50mSj*IGD}DCId?*VXR1h{VY3=1>zH^?tPDQ(UMS1-l@3c9;(_wfQP?wsjOqwZ z`964*1vq!Pu%kjTSG*(k5+kAW(h(=XDVo8S{|}Y0x6(v-!${hL4#e7y&_BWJoyBIb zADO?H3d~8SEjyZZFixntWRRI;Jy{0jr2SF_>7qCgue1ZNWydSE5r>OwMVI&;uX!Bo z-E^FjVbGZQ19~$(VP%EzXb!O5heAteI($oMk|U)`Ys4|&ZV!dZO>-$zTq(}QN~cKG z$XT)h4D1jxk<^D`%UbZJ+c4kgO=_mo(Q{D9Ok(bVr+pm0s|Wm53b}yudjyIK6#6@I zu?1H3`&s#fGO&p}u_2zqL+P1x2r4>P@tl4$o~s}pWJ<@S1JWLJ9$fyPuQKD@KM@6d zHI&+8-zz(%$9P|L)K`&rUL3qNI^euFpd{TQdhwMUx zapGJiNd@45_dzT;g&nfNr&WRPxPo`gLS%m=ot6$tQSfHP$SL@O3)s0W(E4d0Rf2|- zOAO$Yoflt;Sz<{j4Ap^7PXhLG9CF*;SW7jU1K)p|RE4HZ4`v)xiN?|>_@hh`Pt&kh z%ki2ep-W^F>r0hk0j=O&?g>Rgf;bVLR41vVcKG|1@cZo%vHhee)!^qc=xnGF?S?v0 zTbe{;=yT{%)gWo45B{4E_Wum5eI1_V59nX%NC6a$!pSVc!kW{ljuzpS%F`9}5~KK^uty;k|GJXXP%w1Hw3QyLdnxA}$egu-c8%Nhkx= zAphVT{tZ2&pLl(dze!9MrnPfUsWvUmQD)f3JaVRdrg2 z_NATZNaRXApenS2mZcxCE34qOw_^WGBj>0`G}3o5ONgh(<<(e@OyTGGYl}6>6Lcfg z5ys&)9|qe5hw**E;U6fbGm{nll*7O(C+N$`AtRw76CrJqUr|DJTAbx2R9I5VFcL6=DR(k+~;He@!HK?@^^dqlf1^~q0KBo-HkQj zl7qr+zBaJ+iouv*L0~2?6P^hv(ph1I^jcgboP-+EAu@x`CgJQM^k4L1)O0?xn!X_S z=vg9XmNKPC111@-*NaebK2*jk17SNZP9{I#TgQ@K(jz(!w%>!gi3eR(lV~p{hDm05 zn!=25jW3Bok=m~HLU#%9fBR|NCqOS?u3kcGy20C zlZ!a_GhmZFr7*G*p7R@gT!z$3>VVHV0luRqo>wBe12%{v{LWADr_@IBklXNRy~tT< z9%ac5@vt-+dU?m-p&v>Yr3|Qn&B9M!6<@BNN>=&(I4md1Q5Ga={D)~TZIsLUpd6Ubadx5 zpv}>D)ruAa_x33mhd+^dz7nrskDg<%YeVU!r!-w`F8qOv`2+dwM{Y3<=nV1_CnuWm zlCQKk9Zuen2jWxFf}C+PG7y~Mm5qRdSk>?M^GdP8Kk&Z07A3XU=T7XP?3(3Yx?$W_@C{>_i>KJ8d zG`R$CJs7vuZ-@)=(ht~OBvE2j3*b*1A?~)5bT~PkX%6CeZ_;swMX@ajhq2_vMDuqG93q8tgQqbP%3BOdpo7i21K z8D|hFmgCevBor(Algj8~bnaOspQsV5!beQNjb{$x_A-(yNyz3aNMFQAtn-;zS1Jo% zy#}|4XUIv~OOHf!V#3ELaP|-2#N2_M9zzCm6%l$L@{x4J{yMmGEg<_yRk8y&2|sLY z4kAr?T9qEg^V)!$Txr-sW8i!na3`ohr_y@#Kb(#Z=-G-xw5g9*DhqG*1Gy7L-nE?k zhm186_nJ$Hpj{9>h9L)tN2XW^mBO0hQsm!8WS=jkSX2{FA^NvQciJ!HakFuo(j%X# zhSe@Zwq=ppLM?J8k;pIDPGg+Td~zB0$3Dz%dLPf}9?o$U(hiwqJG={r+h=V?%~n7L zyaPGsAf^l3gVV?^%5r3k{H1J}tdr~sC*yeFa$C>^nIauThW3-c4NbY9zNOx(V0N8z ze|G1f;%W9Q@P_z$`pWpT{mTMr!7afN{8+&w_>h;|q~h@J)o?#~CN;&mBJefQ++$8A zcPK2X9@;_r%Es{_DWNw))nRW#V?y1={sx2YhU&EZBRiA+E8XS01yXz?Jkf52YrIqL z%!QK9O!Te(1b%WobTHPL&v zGSDq}haV+=Vpb}P>j#8=jOYD1Yc<=an_!@cBJx$#YUCmrEuCLB#&JfoJS0(ghE%tTy z#|8dFuj)qB5u<~3195>u|44tXx0d&lf2FvcOIFp;4l^_in-|r*c#Q;a38D15vNy}N zDbv1GixR2v3!}${gsWzg$G#HIBc_^<$DSdLK2@9nCs8*9q@MYO!1ueuMm&3 zZxv2;Wy9R?h?se?lMCH)(p8518~Usoi%F1-m9c;-5yELG9Z zqAOffC=}f=ceW};EA&MSsYx~i1v*J~ z;Cz=w=6MDPz-QbA{u5>ki-eC*C2b2_<1=&x2Xgym_vObGrDub{X>&DYG&105M-1B-p@Jx*5!_$bBfb#3ddRjt{Uy_S=fM;4c*r}eA#kZp>6x?_*?yK8{w zlGpDm9?%581xN8d#LDU7a_J#y$9!SaWrO5X6zi2ARnOGhHB+^-bVKyt^hJ8BzJlSH zA;j3%xZ610*vNR>kYLd1w`m>f>Z;X>AF@82nVCr`87xmxY{>P_*t4zvie{3(8t&`TVTJfjwq$T9K@ ziU?J*x(!qo?rLj+Ju^+OHKZ7fhH%4FL#%Ou@w#!oQDYor*s9;J%h0B3o~TwTYbaL8 z9lj19O;}H>H_3n$s;d%fHqn+b`P#`v(W_ zY~w!TiT8~|M}F;K1^zGO4;@G-lF#BXerRBr?}8`E-N9MM-pcAU zS2ABU)ir%C`cPEKbl#L?Zf8Df{@c>Ux(2IJI-9v}x`%tcz6yc(U@|{I@QS(QEpw7< zCSR#IsNAVKrrxM&ueE89>Bi_)hU$i9hOLIbjsJwS4*eNgBdmYeny{f^6GPt^Z|cK! zyMY38%Db{3>~JQF+!t%{7yQ?})jbPcs~n?j7c7^|Y3Bb-$)<-zZ;M)(PMe;X_M2{) zmYbhhy4haZ6P=q}b)iC05r}Yh;C?WduZJA*5IxH7lKriCq?7+VnvaZBc6|6#6?3t}_=s(-8ZC)CL*I?|wgvDp0BRLQish$+e|tW>nG z=(i@O*7U>F%JS0M68*4yT&q0IeOu8%zae;u*WUZCGE)H=$15%0d;)W?)& zT59TMu4dV9Ic523IcY6z-|Q&FxjyCj$EOW63fh7j`6hPFf9vKVZpP~Ta1Uz4G3sLE6n$E~?8 zGm?xIWBAd2i|3Q;59a~9Xq{x4XihP=H2-NXj$c=q%UMoXepqDI?bdI$wT@=4lWyL# z%{L<84&LCCaWmW_-jTKflYd96vc+Vdp+Hkdxee!gfjU-GUK6i5pn0No>E`GU=p{X4 ztPqkFawx=V9Biy$dPP^0q^^Y+Dn1Z->IQ0$yf{_nL1(paZ`Rccq152`F;k1fG+E6;)!3 zxGwT&#TmsEQ;9l2=r`R*Y9ZRX$ce zL*?7A`lz0!k!eqBkLXV6s~Y|=oYU9Rzt%O^HPU`i_f{E{lKg@!fos8xB`?I9Li=D8 zdaGsbjm|-iCid30W7b91-PY6A3)WZGrZ&;`7!jeTbG+-dd$8B!YX~jLGklhiC7!~a zBb)q7DbtNv&0J?h=3h3*PULJ{KiPd*Ir&ogXQ*JTRpcp}DQ_wVs4A)pu&y7PBy9(6 zS8a;+l%|2^p}M_#uWFexUGYRVjMK4G>2#@sP%${v|I}O4bHz2odDSu4QPp9w@3n8X zGmagO1m|RDf%A&{-QXM?AK#{vem(K8BF)EcOXhx#7+Hu^gRVuP;WTmG^z zODqjEu^#NFGU`LGQN{WLH?~>Oa9;tWeIanuxy%gaU;OSORM!=#kkn>7L5F1k+ZlQj zf3p4AdF&zf7MlspbPf7(=Ce9>9rR-u=08;RYEV1Sw$V@%&%k|dE~+t`1f6h~tmtuhlZvtu=0A?R0CF4fY1js~RpaoNK<9mghem1o7 z-BJuG`9IbDG|~dM$c4~0-UAg1J3h%UppdJfh@MFW>cwsL9({}2*Dv}8RmVb_!1M?D zx|A8k^hL#v0(T9k*HQiJ4JC9pw8ZZLLmh&Oz#-HP0>BuTLa9DZP-5kO2sMS#z?j|% zR4k1f`2f^h_K5d^iz|Rc55(%SrDPx$b5H?Tge>wX*7FQ?q%TlpzXv2_Cvf0(z$Lao z|Ku>a3Ph_Zs)ui=kCtSrp@P7pH*X=3(BG=tc|dHgl3S>mokIO#0#KCZKzV=Qj$B>J z6Stx=kSp8~E(@7JWc`9ptROZ9ZZa9j$sVA@uS5p7=RUZT=RhO7J@jDK0bSThHjjvT{IT_E?#H@x#N-1Cp(7QO{_rsdeb(fAi9fbc3%r~L~k ze_vV!2;6qOMgsYO6@{X!u~Vl6>v{bWLUCrfxBe8Qey!=Dnx30QTGSRA(S z9#5$o?&ACKZZKRRy#=T|l>$~a5_Q&Hz*L^mTfkVh!6ql8?$QPI%5)$$!a`bp$0$qSyoWnb$ie}-Ka2?+tu!hP&s;}Z5YXD0bi)zar+`Ug>ch3L; zya`JxM71;p(+ny?{k|k-I=H1gV1)O8+H6FfdL(K#2l3}i(oX!;Z!GgEaE7PwZzX}z zJjTjfVx?bErG85{Ku={l6p4z_<9Mol@YM~o0yx|rek0!#Q2*@!1Z5VU<^Wh(5hg*r z2l71z`YE-r_EkUy4gs^aQ+_b(3K)L7KdzflRelb;r$qI;Vz!=!?$z>+@VgpR|{BojNTUz@Ne zE%5##YCU%`>0ukHq+79y)4-K3p=SCM9v}hrlYw{+i}Aex_30(}o`?!a8~B7K`0o(x zQ9=0pOi_plJ^Jg8Nv zp*1CwIQ(oP>Q4)$`;r0mjv06wSAcTm5g)8XBubh9Y^fJ)c|Di~>rerj4jZ0`b2NZ< z#1pHE*j*kL-V#>b0(B5IeS|x(5y;YLR0ti|o8MKg$>K;<NPeXn)OiQJPNi)I+;hV0berEia_<|qHg>DKClUV`W|LCzP3Qi zsx)H(MmQdr<}cj82jbqV!_zr|_b(TUMglvJZa8DO3_i;hE%#QNZ5o z!#{VxPNl&2RLATCNRH)k&A|5?MWI2jwGZX%EAZjc1@Z8k!J5Ef8 z31ZdZsKxXG|KS!;-GJahvZLO&y#Oaj6xVL!*hfh$!?2 z5cTrFa~p^qV6jVJpHC4Vo1yY|1Fzi}`|$~TULLR00-n7gtqc3Fj8j}&&^($=um0jPK{!!tdD8ma~L+_e8$<6cw@l!!Hb5qC}kwf;)K z07KsmWH=4A(oOJO0`T2w@PLt0zIf$-o}xdjs10hPJ;3PQ0X#esi2OYusa=s#Tmp*f z0)IydAK*d-_!XYo9Lz$|AY!crzrYWVvJ!jN8BeMwyv};m89v}yrDKnG0Ymq}v-Jh) zeGjigsR3V+;7Ns3E&Apq@OpA!nSN9?3y|S{Kn!>a+dL1PelLDzDI)!Pe65DHu0*VF zgg7Y!E`1v7ZwkM!mF%eEzQf+!#rG}fdHn}W{Ta~waQK`iu$(nG0~B^q13%diR@?$B zuZKMA1#CHhcNvK)=mNyKnuu<_aK@Tq7Dp$<)0Q~bHE_ODU?2N{V|PRq-Y(TbjGcqm zNkz5lBlc`1EbW!p5&QW92!1!Iz1Wh(X&Ncg#YSMznNd57M*V4zVvzO`moA+U#?je~ zN9-Q_=`#lc;Fe{gY|Id!N>ol!7a0 z4L*Xp{#3C8>SXH>VP>EDxO3=`L5#wjr~nvf@!%D?P+2^M+%p7T z@gJNn7w$S0!HHW*?|_lE4N?C9klqAXLRZ9@^{8>R2a{kC@`=u{sm3H5dZ~VRsa0eX z>Y93Sr?3p~{z$xwew;7xrKq}-X~>2Qc=Zd&Z)?FPFD8FWa?#Gm3a5o`;zfzTyRO5N zSx=qdTAahzCME;-nhMNcbS^TzM$#ozu0D%Pus+rQB1UcGX%}#wdeJBFc6-Sx#PyZ% zR{8L>i}4&T2~U8J2ZZ{lZmvRhzZ-ki6P3F~WGlG}h0K-WXQ|llXimn%&;1me3cJwr zcU8EF&u_x%=z;ufD{W7BajtZPOlIPs%2$h9%zg()q7)QDMoKpF4pA_c0KDfjz;18@ z6Hk=1cotpBB%GE~bP(=f7ZC|tkeyJV2^X6HlWr!|#d)cMC{{(NhPhfb#B}nI9%5cV zd$K?Gh22GyQ0pEq8EFTmH|3$^IT;hS68Q9hF5u+%OB@ZLmN|j0WU62e&_*tw{)PUt z$=rCZ15*YbCW|pJb)~vORe?&CNCk<*t!TJ72)?tTI6@fC9}iyO=YcUZSklt*;IHY4 z4VC-ZbS(Etb{6;dZftXAoOGKniJ48=LMJehCkN%c0UoC=vVwldSk6lSk}d2I`Ayj> zPGahkS>j=IrZf`Gu#7CR4tQIll!~rzjyuHt4feU0X^ETLI;IBOhdaxjBxm{g{vF=? zP`9h%%MUCO4Dj{q@fr3r3a|^OvI4geikBmBzdt8yPBsLldU9REJ+=TrG&(O-2#)fn z`D+C?ias^~wbb>BD7lHvVnyyMw}z=BId~86;M0V4$o-BBUZEB=K_@a!W)V=s4zlOm zcCG@;(V5aMVRWF8cMQ7ns|Q<($)u)qUpUHF6m?8z=1*pjEK)T|{Y%-Ei8%z`$BtE*Q-z)UB--I*Ks{%VRByf4^({?39tB# zf&ex~lyEAz(YMH}4p3pJ)Q47M7Om3`(~Gh^s-<_f7PYzsaQ3UG_sNCfpG3QWLYDEQaD}vhW3Z z1A~P}LTPcOviltTw}nRpg)a=tWG{9C+@ zEV3^d$E;p^Z(8g;H(qLZ%|j1#Ib zHMx(plTasM5G=Gedk(6Vo5?2RQvrUw@J)J2k8{mrm1#|(Opp!MmISs9*NfSV+k6K) zliU@&!9n~6p{w{@=*~CdhYF3Q4d5!R1go|@(}Oe1JIQAQ!T*bW!L{HT(`ad>7$L-< zcVi)Xeg^X$_>!2CHdwd@P5TJL-e6+ea<&Iuj<{1F8C6+$#NT|PBZ8J} z2F@}8_w}7*mG~#$i?1zx0Pk_1a6WJp+FQNg(YBNJhzMnIqgjAc{DP?=k5z7#-(uEC z<-i7d4^P#d?gl?NOX$E?2f8{#WRW90g%aRQ!7nBZEMjkZQD#v_y@Rhc-pKJzcI zb~AhV@F2!YyRh5ya0j@78)qkS7`N8VQb33oTi|YfM(j=wfp3x{4FRg6MYL@U zwxycR0bjEb{m5=*2T?gOA!F}=^L_@E+zG-LakSJ){3XOmEfFdDNz_FSHm7dm!(TKeq-uXEIp7`M5pj zizz}UoV`IzUv3MlW$FQuku8Du&?Rz7erL3+xFGZVV}?gQ%H~v@|AI+UYqagQk!@#Qj7AZj}!?cN}i+JAl}=MK3}cvV;}5S8bMB z(T>a>S|ED(1aS^I0@lbb#Pd^sV%dpH}bSpBWSzxVK25!C@=;Undj0vo}*07`x zqya6&4Kaqi#vF#H=%YA^+rn@#x54t}hhR>U0k${-PtApFeHfVxe$*Pq478;yIR0UH zT8F?fzu4WJN1@+A3J z#I|>`Px5i{bVQ?%+!3}7jYrNuiENN+i0z@dJT^ED>f`Pp$IpPSwg!xAtr!Nj=~U!Q zbC5lD7iS<3evclEi@rWynO6mkaD)G(Z@%w<&*@v{Zyj7LEW@263D(jV$Uy*=!Na(h zwg$6z1h`oz#lLXh)k-IEvvrG8!1=pM^O%`jSLJ@g%ZO#s-D8eMDI#f9*Qnl6yP~~N zW4)?dEl-hx{>82}wv$$srJzVCEL*s}$ZdV)obH_#j1kw8Q)~r!Rb`&?l46FujBGd4 zRl0;)Sr+E9v~={aQ+sFoAx8xl3#QD!-URPmugmu%&{-&r+t49kVAo+$9f7+%m&!4N zxO=io&|r8aFN29%KX4wirP=(IK;^)8Ux&a4(o;1#%pW~C@=cg7tU+i-#NuKP;{zpY zl{^#E*6^9j^6zzgG1n_9Z~9c2mZ!@}%Z)4;X>H|h6}Tdfr6zWW;)wc?_PI7m`$m0K z{zz&HJ&o_^Exh22cmC^q>+IwF;2h_v>#rKD#g9X6VIFe92|{l{%MT26MfccI{~Z6l zzy;x7x*zi!KFf~EcgRb~3uJ3#OJ$|G&Ey@Q?7!n)<>+jEZ7UPFt9TI6EdG7mt73}i z>)~R=)Z%Q?o6^HmdM9*`=&B-t(v}s!p5=Vc-H?0lXIl35yzYfnt#;SwK#pjkRb-RZ zSD?$lg`79k(|1;HVo&oSzU8Qd4s)(|K5f**`DMWUbKoy~ z1dsa5`3L%Jo@8%R{|9~nEiLoPYb*OIiz_b6Kgf@uA8I2#E?y5-_BC|fvpu(rvaB+N znrb+INgYBum8hNasZ_mEPZDOw)=7AgI534&_*8mhai1YeNU>b|x#rWv&(FUP|JLr? z*i0$=LqT`j9Zy9`lATlT(+n`)3u_*JBCKL~jfk?Ll~f}nk1xU9!SUF}I_jeH_ar(= zuX-MN3f=kc);@i(ByS2XM}PIyKt*4y_pSFo%;xIhY3)13{~{yVWwN%4$BHtFX!#}C zI9Yw}KC_(`Blr1gzGT-PTcTxzX=dSv!g)o#3!0e^2EB&0DebGjt#-BYAE`+t#>5%p z-AR*Dm1!?Zmo8pL^~|>XdzV+0o}GHT@MGM^V_$B3cV^Z2b-~1V?=j7li!{>>?ZV1N z?vCge*)6toA|IO-o~iaTHTVavVfOpZ2(Qa~+tbk9-T9C8MPZwQzNVq}g{}y1xnLwP z3XiZ@R6>D!7VS;0)5Y8r)W5DM_9-_jQe_@mNjf0x0Y`NowB{v$C(m*RV{c_!Xg~hq=k?yre53tP_+|h15xELeH&^h^n8XbZM4vkQE=3+xyASePR8 zB=?xFtX6hZVOCvL4N%^M<#b{9ic`_?zu&XS-rGFRv;b2s8x_qcELXI-=#lBAwY+;% z@TT-9Q(5+};(}tP>^pOsyc8Nq&6San!BQ7fx0EeWs&PW)2&-YX&aM9%-6(c-hpCvO%UIi5w+quJZn@)qEBH>x0!y(=?l)m z9oXX3CNu?mV?1)$;nd0WW{uP;?0~v{Lw=Hfi6_H-#ueu{W4VH9a2w3G&3?-(`zKdj zOq1CrW^m_~cUAwW3Vu&vVV%-rezP!;d#x{1+?4#LRFlL~(Sj~fF4$xG?!Vu}Z&$K1b1D?1n=jfbxjuTH1m=-$n8fu~`-kyHh}Y0X*IC!laLcGR z=4(G=`s0D%3DKoHls)B- zWwYhc^0HhN<|s)=u60e2qe|OXItotI5r3g?l(&nAc(k5{Zm;u{L+S9_O4=@9|LeOm zF$3mjpsBbO{E--0fh?EPvmsO~brE6$9en46W6DO6t&_Htte+&t{vDpL%}{Mq#_7L= zcZ*ySStHacvw2$=R>|~zH)hq%o|gaA{Mx?6eII%=t)&Z$U$#*_+AuA2V#o-6Bi&6* zVR34gsH-SXb8p4zzU9u(P!Dft&9P^B8U#f?3w%K>y02Npu=8}VyoLI*_N^{mcUfH| zFJuQpNAesy87!3%E`dLA@2-ja>P}C#=cc#1ufXf^Om$yzws$PF zXV@n?wz_maHQ!$fL2YRt(AG3yHxCgVr;CMrjo?lHc%GGQH@1npU#g(=iR8QSCnNS6 zmTTwg-h_3Dntp!6 z47C}O4GqBv^QpHh>&vZlPB6k%L=sv?B7D4(Qoq2^tJHMaFy-=C0#A95R&;Vf#SZ^?jx>5_ieB_TSEJ8 zjXyNF3A=olPvqYO%J?^V*LfznE4fR1%LV=rCP_QUeB{BvG{n)uDNN_>7Hkka3`M-L zz7?3nyjl8N-8RyfXe)iNOlGOs@s*AJtSv%96O zDZzZgddQLOPV$!sUcxk#6~T`@mF`oM>|b@HzP2&NcwNuw($oj!8-Z!WOD~0GId2WNu#ouDS_scdH9JNJyKL^?gvw^+2D3u>l=j!K%M23ciHVQefAFTOD86|JU zRcGX6yl~Us!JTT)wlGlp&$pj+UGcUEybu0~?u}1EKx|F_l9f`g(;qZ;H8uo$`MK(p z{062@EF(W~59%wl=KsSje}{LFx4L(rr>y6wXSa7bvQ{g!@SX-nL7PM2pXO`e{mXOF zbH_Ww-z#{KFOOcYB~lagKixn_ZGG^I7~wr~gwO76?h=8C%m(ex$nNpGN;FNLk<>P} zZ`7X9xyFme8^*&STSJm`Yh~78Mf;`v?mu1M%-_0a4a(bW+Tb|o{T7@ht|9qsg7UKF zyUu7Zp<8^d{+T9R)ka~IzviB>SC|Q80bj=V2(zy1I&_ZDjuP%I-ceX-@8IKL4PGO3 zm5P`#@*HI^^-#?j%|~^l>IDX1ePh=!k5PLmLpF>5@^diPdxvk5HxCSvzr2EXi0`Yf zs{a|30;YR*xzD>^Id?c;xte(6p#Z;Gj08%38Fje7fhMP-HZ=%%>@_K0Xcp`kF!95f z_i9;qpjcw!x)Sda>&N{Uy(g?Ts34L{rbOwL*PtH0^J zt)zRipAj09k=zPpvUZ}r92lDQ^>wt1R7r~2vdi3N?moAd`-j~`X7G~N>0D#$X{lgo zX{+G6>D$eBLzj*R9EtB>&`)4j$(||dfH(JD+h4mlCc z{ybk*-$CyP?*=IT_41mr$6Gy_o-WWf>g4$VW_q|+?av9^;y;PYiJqyzK4hzM`!TQl z3GO{D#MOM`KvdwfAmbF;?6BrBt>gUhn_{_`84*W9(u{45HH5Ml zMfo3c*5|y-J67=3oaBgrn&CtLfZ$xQ3X>!&QuI^T)y~uzbXv^SxFh#-ZuUC+k}b>D zqa(x)!D>Fv-OSP5KE?6E_0;<{uvWM%nb89^0H{nJ_(Kx2h+89HqFkkdz`Xjnae&3PAsr8Fdv6{r*(!*>=dk>N}p7pnNI zyro*NvM5h03S{-TT;}fYd${Bk*6|_1`u;)Cw#Y=ELru>L&q^?!uDiFpoX+#kjbQv& zasTpoeP;qHzB%&Wo010nw4U5a*>2o0^z!3eX(m=GBNj;2xyS19A)TX6$F@kA9UmV% zF#0M~S>(p%`U5&ACSBiUy9;$aHLVTvd!zNBUUphuufm0vT8;{Coj1}yh95`fb1#*9 zwF3=9Ftu>FX0p;K{};?Ip8GD#kiV5#*&);~yzn>ibaGLb+f~Oi-fQ;_3bckoRwVj- z%1A3nHD)OHNIp-wM>SP_T3tz_ECxTpt_~2;nP|dvd-0 zM)->2sfpJUl?mC!HbxjjLk%yqR&87DUDX!x$6b13e-Pn`MA-Oe%2@y@l*@~&C#9o}@z7BGq}rG8+ZOhL8wFXj$YoQ+`DGe_t% z=_;yM^+||qvSu?>N%M) zGEyU<4m6Hld5L$Kr@1HFJ<%E&vDcQgNN(Hu*JojTvTI(icVulUWaS{zW~w??^+n!H|e5joat0}Eb{S&?_tGj&UP3np%roBSS!w&X=s9gBCB#0jUq>0SDpbi@8aA+wjx_dHRP4hEtu$!_SSOi zTuYpNos*nhorzAX;}BTF>zqejHKF*M?0*vI$uARfMRaAOclQBnM{VI46Cih_mn4V% zs2pur72zm$EFnFKCSH%r#e}7OVM{_b84Hbb4QI4gWfO?bS4&q{w+GDil6sS_uePCTjx2`G z=2!a$xw|^&IBGc>I377NoC&V&u2$}Oo*_OVa9fy1dZ5yOP~IMxPJiUl=Q*A|iR$AD zs0E!AT7ic!&9f2uc-I`=9N`YD{h|G;y@VqV8h1b3jl4^JFEJ~j1%F(qBRwNd;4Kbj zBKoWwqvJh}Ns&L%3<-^mHpX2_ES+>Zp-$ZFm|l^M!`Ft^4Gjs6H8Q#x$~sI!aFw&I z$&*((=Vtck+*kQmOd{s`Hg;8TGoF3uVn__+3E@l^`57qk9o8|J1FO+Z(3aIU({9zq z=-k@vnue+>@{>%cv^dz`=fgZQ!I9{=Y(HmzXP@AB?#y(r_4Nt97fO*t)I_>+&p8jL zlhu{AM?axlmc?qA*HUlcRv^}Q%l*yy&av6C#Bl;@&jf~3J>KKK(7?dp zcfJ#F^7ka0c0vvB7WgSv(uAF-(CYSu1)|Hx9Y}O0_Dv`rH!vneLPg%~Hyq1Mk%tx&I>@6IH=o32v{n{q}=0am;vi!JegSJq& zOmEiD*N@Oo(f8J0(NESV={jikEBDIoGIOPW_!Is#Z=$;mw198gVr?yLJuqp^<=){d z9o!*Ylg1@S?)L|aF;ld&1Ra@X3`fvC2-mM&fUh<&^gWV33Fd3L5r%I zW1~aq?BNuh3!t?*(r@5vi3Z^GTIM+XZ*{O+DzRG?vHE!tt&7Jeb}mUv-c76=w-6teXCuVxg4k$u_Hg*?zA7c`N&1p1II%o@86?xahd#*y~U`raO0e zh6jd9u^becv`6(x#_z_NMxSAjVZVNop3`S(A8KONjg))kv0O!3O*{l;)8?*T_7>Jy z%RO_EMG5ThkVoOKfLh{6ajDb?T{Zx;J=N%qIQnV7q5LoZMh zC<0%!b+C7F3Et`aO1eb0Llv(pZ}?^G5|U}WZKTG-=n~tZU$1Me?Wg{tJSM-!ejpm5 zk3Y(@(2-}oXHGVCG_5h;uzqsXaR1|->Hmxq_OH-Fd?h`gp_rV!LD>d)R}3`JTQftX ze10i1rKeuEC)QIDvkk^NcG|=3#q8DX{q40J@10-V$-e4=MZuH&3gI9ys76o}8!UxD zpSpBde6foOhm+5i`jGsgMBVtmiru zGdNeC->D$JAh9sN;7Y-%!mouob1VB;_s76ZX&(DQ@m>9=E?{VfPMOVN$HRt%Uk=X; z`x)vrZqc{Vrm23&+A(Qj-M~PP+}Q;)M;4hF(>haaOB`ls+;{eLCwQ-74#QENNO$N) zPNPt$KB|VP>MEns85qYWkWeOe^+KY-6PRiew7oPJlyJOC{yv(8^duQKyF^0{QzgIPezwM`= zYnT?I54#h-J<=W7D)M}|BD9U6jP@VpQm#FTmZz3o)}FT0wiUJo zHlsb=(cRVG`y+5p9L0>04O4_Emnp`|8_8;OquImE9aQXQOSgqZ!S6nu=a5tBNVk`T z_IMRXL+4P}7mX{yUrcH+SQ<#qlH zXxV>sT(pa}@tFD6-qO(gz%<;PY`tyo;~wkpC0ro6VCWoUGnsn$ax=BLap0(Qkq?CS z_IhL-SG;B1JDo{T`>gB=gW_f*5Blxgcbq$XOWE}oI# zOgJ9zjJ;PZy|_2-ThjZK@_(HC<7wGXNzI~4X}?P+-F8c(B7f0fOAFf}dwts&Q>*-) z*|C`$zfb!^A{s`YiLu7$WA;V8iYOI! z)G%EWrBJd%q*44E|6g7sX1ImghgknHFE4sj*t)2_d8qA$GtFxV6z97`%jmAq5no2A zORS@v*xsB4lRgizZcNkso6q%^@$L4`^X33y$@ES0&-UN)eev#cH^%(RH`eyn7;C8Y zwd0hpEa{`#XBZ#8JbF=d$LLbgF;TvlsQ4Wv&X?_zs!csyHY4$TDS9$M6(Xe{P5O|^|d{M5J~ zxCeR~d4JD+OApwPW7Pmk|DHYo$9oWZ6SSlo_lNLv zT%|E>G1d0cqO;39S9y)>tywSJF*KY6L z&?B)6eN7*0o$UNOx@pYFXl<0+`PR7~j>_5g#qhosGw(7G>`&^Mk|+d1gZ-5}?OccQ z7r0J(*7)8c>3jjsTxr3r!5g9aku<)oSW4~-ox~7mj6KRH^lFg`1ugVZ!iDYRE%N_{)<{o2oR*~xj)E?k1^XYQI@ zD!+i|d*D5ns_ds0Yv$|b8NZu5p$HOw&582kVz(+%QJ-B z$js0+P(dZ|F11+aOJRNA7C07s5qcOd7HJD|w1%6?<#N0DA>t%>cvIBDP}x0)@^21y zIJ=0K_u6(vYX^D_l*xP6wO34>J6Q$X{Ei#nsj}19Sti?EiIQUgDk`0;hSyTr0b_SMHM03 zvQzFX50a}(PC>_Q3pWhC57Z0<0t-XYaDR1*tPOL)KLfA*{Q|{93nQ7x_p7OF2Vuk_ zXG+86AxaIREOmlj#}tC5X)v{nSgM?oc%cdZG~7Mhk*}zvGga7bY&;SW-f1RkKj^I1 z53xOx`lk9**C)v_8J3Wa&=)tYw`D|iiOG$wZPREwh*$j$-T%2Bcs2x_kvfsF!7koO z`D=2{XT8bF$_;sL1=ql*Tnju+My{pil8cx>v`yIVy5suJhU5D2y6fy%c89jICQPqH zZEQDA8>6VVxXu}1O+1cl=lbwo{w5gX!vq6YH?$OKycL7fL$xALxn{fz>1l_!_563C zhtvy9=wk35I(-$yItsiww27(53}aT{>=Z?ZsoV5zroYCIT=0!dVWum+ z3lxt*pkSIn%=xaw$d82ATmqNQeL}^uS|kSN=FVX!I3D%5ar|E4UvZc;RGK2T5;sam zlm|p6ISUFe6C8?F)b~)8oduV?2a&8zH1;>OFj#b5v?a71b#CKHI}_I~K_5RB&*%@+ zAcNW1%G$)a!@1D5#IS^^slI?ec2=agP#en5F;YHXIlR_?&fC@J3>HOt;plMta8+<* zo#3Vq3qL`>uZh@Iyoz6{*iTMXhoef|6po7&I9~RW4ygEkW6CnM=uC1R>UERh94QX{ zPGz;XGE)8^6_maSR>6n*!4q&hwP4>~7A}f6q)AY_tdK83+wfJ|gk9bSDAj8~yIxUk zO=xHZ9k*oTBmFD3Clo4^bsJ3yj;Aq`;!4JTcIs_6P45itjYTcj&{5cLPSKsAx+`{c z&*}?x#CBp6UI<0}YrO3|b36&Y1A#%IA)!CPEPev5!l%%*$O68+$cv}MZcr=yfHuw| zS64ou@;Ze43YwCEZUeQ9i^`-vfz)^us^Uo$Lrs8oQV%WfC1R7Bp?D;R^h7jB+obvO zCD09D;9Il!-4ar)bV<4=R|kc)81!{EkS`Aq51=vs z?FH>&cD=5&zO=rRZXA16J4pKgPSC>aXzetn6a~T^IB#{}SZS@&$^kKr4}?F5E`|z* zE1{CFz?b`tJIYPq-|>xvfRHS$mixmS6oxWyFFabuF&@@ZW2m}R9z0s(z;DZfYN;ML zXH|*%(6FtB7A+#B!||hsBPt$xE*UJr`*1Gohr^>WKB*YEqB&#{a3c3XJ$V=&nw8MA z-B2^oCCGyxVK01H#X<8~t#*MUDH-0NE^tsZBwj;mh_Dt$pUvY!+$`{l55 z0^}VT-z7mk294EFumt~B2N2CsS)_4#-;Td^0{P-ap;C+gMQ|hrp+`G`OhmPZhm$M= zUZ{m&JiWo){eyZ0rQjHNI){MY*&eFNJMeR4!1wtE?wf`1#I}aRv>yCiQxzRNEs0Qh z*M&Q2I8;#Uq425*uh%yC@H&FX*bLs2&Cu<)hXeFGJZpF1SQ!O{d{1cHXF{pA9=iH> z@K+^4yH*aGt{~}ycTxmBXs;?OOYu|6;c3l)Q`HXD|KIp&N8m?jh?*gE0#L5sR?4DZ zXNTv<0e;W~c;Qmv*I5X<(l&Tk`a)f|3GYX1xEpmOPsGAoxEsD&8YGFG6uJO520mJyHPU8@WJW*4xm7C_Cq6W?Y%6n%Z*8@+=&Ga3F9 z2d;k`G-Jro!B4r0X8`pcyeD3`f1=<{y$VlUIqo)tS4jdunV-74^Vkj#D9GouBGL02uNyM;t<9NLw?5$ z&>g;ucsNlf!6Q@>)WHtWYJP`vFAkNlCAhBkc(+F4y*!1#`2l|yrCg|(EpUl(s(>+B z4o<1v@QyTvHZdLVsRz&R7W_mnlxa|oHU%@dF^HhE@qHDz8m8f0Y>GYGH1aLTlWsWg zDnr8?fuHg^e2<&(ZfC0xp$~lp%IHaADo8v_<-u_CK2#p6OW+Bs558}vd>dWtno>}T zM*lM&9@k&*cu$qWp0_%u&=k(mMEJOV z!ozZqc!Kxj8a{6|eAoZLi~1ODjulvghGTZv3gzrt^enB!RXqDDP#Wr?_ z6B*=IP%0~dC@U-7)I1Q>-r))pNS5@&-FO_|yBhpY2Qc!IK!(#pRXhz2F#+0GJ7$3( z-tz>|%BH|!eFy9261dDPVBVQAei}o2UJL%W6SzCIFe>NbyHrClm8nJp4SR@J0)`ri1EQ{5pbrRSu)|7@TQyz+J1TP6wgv zKRFs~WHZ)~)9~^P2e&Ms+{230{ zuXukmP}f|88LfcoQNF{K=!VWU72o<6^y2fe4!ngPb|IXG1MqZy5;dT0xdz7PXi`f; zqmDQ42fWO!@Vxh8OzgxRD~oY55;Ig6IEfmt#J+6ga52p)g*cCS&yf z^42fG{C5^i*dh3iXD}LSK#SapD1rI@D)h?f>UpKLk^=_YND!PJz*+KC8II>y0s3MB ztmy%G7veBpPQjUVmH6dFwUOuWU8}$mwGl464BX$rWCXM0c+67kvF@l!U)+mp@K%n+ zckGQZZ^8Y1376hEtVC^a#WZ}jEvUs*8&nCOf-brr->V+x((~~8AAqxQFT8C5IQKi? zzQ4vyFdVL?uTa;I$6fs8*Bl3@Ryla<*JDgv!uv5ED%}_G6wk)WEaPuH1CvgJu~8Ur z&;uy!-MCH|`xc3k69S^~&vkwfyQh4q+!MVGM=Vk+8ZZBM0CU|5& z6G>z$>BDz`J8WpWh9iWiPzB9q>FFz;8GN&+i$ou^-+< z0&kuRGiEkc^&2=5&Vt9RGQQWZaTE`J?ME=7i({W&Piq25wn^}#`r$16h_mi^JOdW`>|FdbH=e{U_}3&j z)_1{2SrBjKCtO=1JRe8lv}^*`#00F2%`rm}m=o;SVeP}J7Q&NRhr9g%EP=szs?+c! zufV063}X;K0tF9-O(_g6;@>ul(xCLjGHhj7=hdeG%5;FO57H|djD0$G|vG2(I$9Hrqy29}t^<2MNgy_`3K`-y@{^S;&1#|mpP>FBz9y^EB_so8nGWi|>pm#J5dS~g=nWT z5~fSxdepio-do*A1iU3s$KNnZ^zS-0y=@o;t2=O&;gM;$`Xr^MTv~Zgq9M z`){b`N|xjx=F`{7IqEC`6#$$@GoX*6cP@yd643h`RJAy1=+GF^mR={}gWH|WmF zO|BMMh`mioVyZM1ZqmN=apJ5HFW)5cs2F0pm?sue8#248$Fh-oOia=JL;t0&;dA69 zrYCcQWI)4p(n;)MdZ(~m7(>5fcaw+tdcvRdR%W7FMlK>R!Pxr^d$Hc~JuxKvsZL}m zDVKjqgxGoHE%9&R5ZOeRNS#16qaN9bxkqhK*U1!dnhr5fltjP2|@S*!yq{wNjgaA8Ue>y_YJ2{`rxp zMvtP$kYkn4azUarU6=ett=LQY>4xeYDO!C?EC=uUl-d@I>v>=#n8-|ZA2>55F&AD@ zGO@#&idkok`WqM*zfl6!oUTM>VE6M1jN)}TMatN7sS1g?z92r=sa^yVppNnlj;hYs zUDQ(NE4@Js?1;4WY`9A>C!pH09{&4rn9+)2pLdLqm4i6zGy#X#rTmFCmQ;@sg{W!d zB&C9MU4Dt3cxyFPd95BGx2TQ4zR6Oj5gk+=*6&VOx2BR_Y9aYTy^Bv7iWOls_LwwI zc<)iSFN(Hq@S9)DUBwXRL@DE9ny zQlk3Mcd^TlCjTH7V_wY0%y=04;LgAHTi9X$r4)y1bR{ZSUzBU`qz{7Q>Nw`#N^tN` z!WvFd7vN-jg#Z30u^wlYtJq@)4dpEq1{bgsoR4+=1gNJEmBsQ|#e==a zNRVv4gNvdE)n+GZo*$5>&;lN-&p5SIfm6%?-<}tHi41Un{*!-#nAQ*L!4a&qi*b^j zuhzneV27HEr?wK#x3SnGHN_6Xiv9U;Xt!u;BhEBi$X?W2_*9drc2p{zLsw@uGaWTq znoF8rb@IcS6cC>#vZZyRZXq=7R`w7a1~arWGlLnK5QI9D{s?Cyq=^Cxl&%*qP+jQj&$4G>EUaEPNV{EIQ2)c!+cw;>HEKzWh~9Qb$8P8AsJ>CN9WU)0ZKcdVbXIM) zCXLyyI>abc>RjP}gQ`E>@AQp!f6b3_&35ZNZ9V0^>%FS?qPL&#P2ga-Ca2+caeD-p zv|Mf}&xSrSiCPSwS4HYCrVv!+W0;=w1nM^CDcUO<=o;F?XRrY%_8V&s&-mD-k;GHwG>RI|tgiOXf!9 zc6M#@(B2|molooE;;Zgk?|+Xf+VF^*?;yqDEIb<8jA!)U$oaC^-Go!V8wS%=TO=YG%)fswrEtyKJQbx(?vRmxM z*F;*QCbA-YEtDD5Kxxv2JIy_X;=Beoh3m-&g_2T&+(wy?UFT<|ncNVr#=%&J#!B&O z6&-2Y?YxeZ@+oWw(@QI3-x2+?fSepi>7Q`O>9-%W->|1S*4hV$3kWR^6)Q#vDbpm7U9W|Lcglbn;?ERjhVsMywZCdBp zY~83mMn~Ar=I0i{c0OiG!8yt06aIA0w-2$O!_`f*$69$~7q)|Tg#MX!pyCg8#LwB_ zZ{V5a>h3EMyzMELGxg`+nPaj$-T2VD z(!trDnv0qmn9@wo4X^ae^_jZAv@hsem^&>>4{-?Z;MZ|W!V%<6U-ompgPw)1ZZ3l- z2i(0A{@(+`;MGAsVdOBsO31@bX&!bm!@ztfL=R(L)6rBSR)ia5b+V2+7OJj^3~kJ@ ztTNWpe5aSPA=7f}IQ!k`&IRI<8YBdx+B*JmymY>BijL~GB_>68K(|6qv7HH$&k03` z!v6N2Uak$E6W}CP&K;0B`Dd%l?5v)-Lr`(A0B!h?j@(lVJ)qv?3v6~j^E2y+jc z&AHC`1KD~Xtaag9NwaP?KQ|QCC26ZMrKxS|G^vEpkN=aK8SW9>>c8y!<}Ks-mcJtJ zbKY{7)?3_H#FyzU;Opiu9J~}tjx;u5)?>Pc5)HZcY!hnh_GLY<*1DuG*3 zy{kueG-O$Jn#O9gnUA_@mL7Jqqd?4;0zHzF3WTF~IX^j%MD>q4<4m<5G~Hm`nigym z?PKC6A04h7G6afvI_7V2>Af4>FLHfZ&9k~^UCFAS^E5xvD|jpV{tYbQHcD;4raw$Q z)qc}gFcvfiY?q?UG2WISF z#3%!$LC?HixhZ)!^RwJ%y_NFGt(0Pj`EZXHp%&2xnUiq&2kF7| zUg|4qNDI`Bid9`rPt(VluNW({*_z4vGv@79$#yO(5HBZ+343G0Q7xm3M;oG=Ii6Uj zo962h!7$9%hN*kf?_h-255{|SuF0-tp7EY|*VWt+IjK1xvoB?T%5CGWLTK<`K04=e-rETYHIZnze?jrdNiCY#UaiUuEow9-}=fBaV;P#?_8~;?z4O zdmY;gOGC5GI911KziJO@MpG_X=5>+%fjZu@;3#K#mV4g2hUCA@E0lLHXLZi*ymoHR zQ_lBSU{<81v`Xzk`l&XWb?i4?bwfo{X=`!EGiRr$uSj3qYr9~n34-YleMNRFQ;@z1 z_x1$oE3fA?NHM*PWZ_DlDjvVPx0`eAaUF3@a-a8P_|E%n!9Jm<;l|ugzOb}Zd4^No zcj`EvV|yg2{h`g(3fgq2N9K`@hzvC>SI6$jOO0fCT?KZqW{##E+eJ?p2bvYz$f$$S zY;?Y}pmPO!(uM5Tti8=O4HuEh?!uh7g4sw|aiv$nodYG|obBSv^*;05bvJTnxn{e* zxf*%AUbpXze_C);WVrZ2DNX9>TTBb>F7~ue)xR^&Ft-Mw`iy0kWv97>X(%*5o!Qo! zH`G2NL8X)(k|IpzOL5P`1(5yv%zxK^%RdL{iAVg`0=J=^91@-s>Ba5mvN;cbR=kQ; z5BY7NER?41kZs|;PJ@!N33>)z>;q4tYFAiEkb6iYB&3NUS7jD0(<_?a*% zY-cuFuYggVZCwT`^MBS#wma4y7Q3l{;g>gZlXjTq6%Qc-e zCQUb{9P%0UWFb&h!pH-$OZUaY;yH0MG7XoAbHse1o*?iXn7pUN<`O06%5g}9dM00% zrz$Pb2P=vyeSK6OZh_n6Lk}nqRrbk9XF85Fup{`DA}32t#LnVq@h*Hmr$D&ylC`NW zR8x8~(@FE2b~IZ<&l&zPJ}??hF4GMpTz0fLEg49&tZc~Ch1qg!TWv#42Zp5kkn6CY zHOsHXX2K)xZlorEqgEzR|7(L@$W`ox@1Q!-A2p^w zQHlS8y7fm?a-X73*A|}clJI=@#6LVzNs7Uz>V)aU!6o|J+6*8#ugVZZ7@g{&?rUICnEx`FxC3q6JyR8LFc zw6T*Y1lr*h&?zm*Dx>IDNO8MK>zIO!ld)kw|HM?%G}PEN0fu9);(C9AFjV=XqVP{kYf7SB7Lj6H2Uu0QUp_*HPRJ-cR0;RhWLT&ZeezuvC zEl)zFbvhjB4P;)bFRzx@$u4PtG#qyZY!*1b??^PI*a6vPN;j6s9W06=PCi}<6Tro&*N;| zk@yoljVd@}F2b9(7oRg0m46Ltu!V5O+<-dgT|$HYl@0lN6{srI7G&Dpq8R#5Fn{VX ze}H|`8E&I(%yqE2zSDc*O$>o@I}~TM%}9jGfyQMDD(XFPPVTF82NkJ-?3V7~IcLL@ z%OX8GOSmsA7VZk~g`!}qc!lv|mRJd#gh_%<>@EI*l9*59DER6Tz0{W?(I09 zk6z(7JlUh@!Hp)DqjP%?KFM70G(h13u`&YkT@iF}i-UYuilnjE9)>r&6{^l{@aJpL z!+wt1>MP7K8}WQoaTn`i#_tO@*E*c`tAkxP8kLKZI46hDeJzBl`f@NF>=ZY6)%gw>9_e6|-9?=^h?I$Da6eqamG&byqFVclX*mEj@*4P-R`N2& zK>%H#-l%&tMz8BnjM87e! z*I&oKlm*RU>K^LzU#%($}D3fr#z0hs$g}&%Luu^1HkbWXJ z>Ke#O9nhid3hrhUK6|?ogC}|ry@EecC3=USxB*@FDtHIhqd)u}z3;}jf;9Y>ChUCb zVr(2hm*6uP!FQ1_+d)XOT&km4fF5tXeLn(zTE38yiS1a(fs(Es@+!G^khVlxWmI|Qk zuED-x9Vjh>lzZS19l&Q=Pzju*Hbsw3Lr74!3+Vq>Cc+>b9Yt4W0@PVm@RJ&#gR&Cc zu4R}Bx?r|0glAisSc+ajCE_3G=RV!b+w`KlvU?B1%CY>y7jVbl!QqaN5F)y4P9c3j(DxY-UVHPQDR z3uWbQDV8V-W==l#2)EG7OOf}=KX6yNDKo*N%OQH>8Z*?>@({v+4%$j}7gRX2l-~GO zW62%L9HJA|ME!t1^me5;*$Qu6Tk?g{03PoKV51$xe;ZcTsyA`p$D=BIvZp5FJhI!i-!Lq|xn)naC$s5&KBJG6r*EIoy$A7$X<3e;JJV{I(jD zUx2BzTlq|Emj+?%-bZbxIx2FVK~(Si|M#M}>XYvf3zf;}aj#O3g9p|DZ$mSXkkaMr zs7Fph_p%ar=16f_ofhXBOZgm3nfvk%> zFkY<*PH+eH8#!7XhS~N76kZ`kgZZNYp7X2!6Vtw0ESGo>oEuqyXZ zr;w+h3Uy6+)k*$?xFkn18-pr(!cr;yZ%28N6RiHO*x7x zUvae$YQlLSVt&DnU^}`K4bR}5;?p|Hfr@#u|JyzrrQd&DD|8g z2twd)sAYD+uhvREgC6D`IGh+#N7O`@)SxWJnov>BQTl+8-Ua>X!_d9Vz1B0JSLAPo%=)_kO2>7=L^4cD=#Qs5DlLsu z&ZrOb%|ybBo$PbDJ$l$VsI&LNUdOK*sLr0{AJ`Mm5@75Na>Dq2qe zzgB3FTt{V*surWpResHX1E4CotVpUB9Jz7CNVO}drZwTjDn*dgBXTscl2So7&Y)Uh zEdDy{~y1bKP3DR&hkEfoOn>^ zA|wb&k|h>RS1sS{Z$t*62R?u`}}#3#eISKdcZr>MF_w-d!T<4g^^Q z>(q30AvGNYIs-B#Q^_%MjPeiqvAXgH|r&tBC`V=hx&F*Vp$QL%w=lsm&&BmadCh9-xq1e^QS zfIhS>Xb)T1DW{Y*? za?*6+8C-2;G*3Wih)I(>YlZtxpNN~DsAFT~7bxaEkYjj%nxyDL2Am2dIFjpp|ioz6b zGj}G!Mrwve1zQHgf&PKh0Zm{Ye9u<`%Y)%i@yG~d*`4P1NQe|k_zG6-}t~zlx zSdY}_0@N@j0dA6(NL_hPZA8y46>I-Rd5B~ZO@f>E2o3phoR`~;bI4$RrSM6TaU$z2 zC989>EMPZ71~tnzuPcJwRhNa@Bx1^$6K|w zGR}jJRZ%0IC!Lz81&*H%m-TPUKc?}9Df+S6+RS2#$NXGe{vx&)YVrR?%7&+eHUw)x zaeu|v$Y0-I+;`Yl!#_XJE|?Q+8(tYvxkMHA|XYfA4B=Q zgZ>BxY!`YU)fm|w7f6FT7L|gV(thzQFY~EF6+SbPz&+=>@Z0z%{Ad0s5@}Y+3e*Yb zQBUZGj$=9OyuV}aUxitG36wWh<%~2*Y9@D8Dx>!uLvn`aK7==|ac*hkuqTWXkx7zgSEF`zyr&eKR{-vby%6G(u@)hANcnu_{7d(+Zk!O+L_>w{<_-`XoFS;+j66(vP zK`6YarYresDK%O-LS?EtWh1ye8=;6f00na!%-c&qc<9Q!g6ce(P1B#&bunfkLn&lw zYO8PWVXJGOY1iAE+pk+ITjrYMOg9bJb)U7o#s;3*IidyL*S?_N&|(qcI{zxtH1Y+M zCR3!(XYhO|COjbAmFvw9GYlY!5PAWn@(T)Jhm7k?=^#|)vkrKhU33jPhx(3hJVzNQ4+OEeC%nyd z`2k$z$j4k18nq;`-yHIK9< z*crNm#y5XMLISr&Y<$bV;VC zCSTi5f5p(u>@$zH*0b++{Oi2v^g4$|RgOC7Y>Jf9DYkzt_e@j4N6BXAXkJmX$tdEK zQUE;PgZy!jd}ank|1tj$?-NgJ&tdms_W<`r_cBj&?*w1ae?8bSZ0Gt21Eg8i*M+=vY38Y(NV6Q07?up%-%R42SP z)G=5nZ7DXFqV}-UC%z z3H30}Wzp0{P#o`T&*|?;PzsXg_LOY~5j=Zgd(F zb<23KNr^nbp3ezs(YmSf+yZL*#9nY8yRqS zup8(k|AoC^S?FErYK+={wLRE}teZW^-e7f@4Qk`p0ZlYh7Bq&VzZBL=NvW*3iRZXM z=w9s#CIw~&oPoK1x3>wX5+#DeLR&*Expu-<@Sw*^ad>t`pcl;`CsMy7^Jwn>Qy(^w ze?#G35>>q-pr(~&?rXN_7V29YQ_XFx)$B_hR~`RG*<&iigrXNjPmT6Q1)LU#Vyk3* zX6|b&iGIX2%>}wOlsv_i2V#P-ky{l$7d+*k>nrHZaaVC2K^mzoKa$rzzp!hC`x88J z{Q@OJog*TDOB^7dQ#+#nQiP#2I<1x+!wzLvvJ!g%J&rVXySAD3FU@{tH=RTcAys6f zQF2M~GrtnNq^F^jKtDLis`yLzXTwFt2G-z8)52dkCx2BeDvMHAr4on^_n{+v2m*Ck ze1;XLh@Q}Z+$R*Ao$i4O(3L5w9mx*YoiRAf?=cJ0j`q%LQOPm?MYoL^7xPC<`RF0o zKU8+GwoaDDriF%iy4BiLrYD6e4|F*T`7M!aAsR05W1gVP>e`uCJNH&jt=z7;D{}Yb zm2o*di9WYKBeXw~CajZcDWj0-_kcENELxJi$PUoG(xvDbeS%)qz1B_EU1SYN$t$I~ zOJ`8GiQm+Xpm<;B8*n4Tqe5}P7wC1o@c;Ck_l*XPWp%Ji=uD_8SD0@s#EIL*IjDeH zaZYPNG{%^%fO^JtoFSu7r~H7*BZvL?1HHi`$Bv6P#utg( zAAc|IX)G5bM46o1Y=x~YE%glw+d_MrF7k`+C;i6H57!S(_Sf=^b&0v9a(m=d%Py0B zHtSpVj=YBXOWdP<+k;8S=o>BX2I097cKL4h2@+97{TuyOeFc4IotJIL_RzM$O2*L} zs7ho}Bv}rX`%6LW?Vp6Fheih9_@Dc(AboX=ca3+PZJ_H8%qTQan9XaPShyVQNr{!nG zpYwmXei*Yak7C|xu)(EUE%U9z_t+n|e)xXw&o<>3#eA_a+*D|)G^NgIe;6_>lzon4AP5T4F-N0o zMm=*Zv+Jy#O|SLMSW;uAzACZOF@9P^i+Qz~Z--}%TjRd(n(OM}`s`9&o!ocbH9gxs z8QyqbzVEld?4Td~hVD?5^g}AgeVitb69bXMQ4Rc@HRKS~cl`1Usi8Pe_yVQF=ZID~ zqzq@u>fYhm=N?);_kT_HT~)&aaMitXR9O%gnP4GuUrTd1|XVLW<$vg|Y(G0_*)QUng&tdzAaS zyD*+;UF6a3gCF*sdw^$@*XrLNC=fb?ER>FXIV6hCh9;{CGU!Kvkj}Z zB@ZG{&3$IU4E;_1k zB8;eVUIj{08YXYtHE*_nCk zJ&S`=xg0T0>7f2WZPKnWSj`vBCoL82ZyaA7M#t~AZq|?HV#fXKA!ZAeLheJN;(eTh zT10M!?cs!AG5;v<3r}lrEpHpI2>sMx?^UnSR}Y+(TK)v6O@v_WFc&!nrf>(TBRWuD zP!Ij(^*92ZU~^Cw%fN%|ls`#E^ki=0oc>g?YWo`+nogO##=mWCqlYE5EI2YLI@ybD z@S?O0X(bD{Nc$tjSnyWNK>Kz}Z*w)%9riBi76Xxt@WVhqZ%lqhcCDX>zYqQX`uqH! z#;o<(i*i=w`tv5bNq_m!XD(e_C;vwN$Gp|qOtN{1<*wxpsCT4wpyi{PM~dWWeZ20I z<|Iu}<4^@DBBXG8pwashIv8vaIP5#^E${8_neATfdYm81@9V1WKI>`gs{y}yN@#kh zPxw~kD;EbES~01qJPN5mRmdBp9<}j5;Or_RuZJG*JaVZAOV}AP6AW6*dUHo(IiqAL z>kP-uP25<>l=eKWY2i_rhq6+}rMycjmQX(Wj;)NXzb(PK%@D^llXJQ8p$)$0?g9B% zb9d$p%x<1_G^={{^z66U&TMNopH1dZ_cjh!774kB`kZJ*yEQl259s0#GU`lw%snll zrLJ|k*=5LQXM(0EN;S}?)iz#GE5`Vu2E!?Vzn>{;lJb1!kX z^4k0j;ZZ&u{5_n;^%R2QQE7oZM4pRI?mj3n*P<>ht6^dj=3XltVq4_+U$eY62p9Eg z%Pak6(-o^^Zxx%I7)h?5-Z;H+O0DD;$#s+KrNkyJj;kBvaoC(%$1KY+T@!k$+?c-` zoan9LdF^TEQS-C2Z9m`tJomFtW>RM5%tu)z^FF#t`JIu9d@Zp7c%E048{{xeWqqcx zw0W#qH2q;Rnm-$d;8fI&eSp2e7LAP5%Yx((Il{k-r?@XMV+&FDkacuc z$~;fkpIJtfkWcWJK*Sgu(EAFwmS$D{8T=W@tex{ACnI-P?zh}sd5XKB|4HaZ)E)W2Amp>LUTE z7<`N4;H%42_oAkMo%*03WU6Iy8HQ_G7~WeNIk(1*OiWLXFMKFxBxNe+2=5Pe@^AM%$+l%4${v^7#dRQWUCyEGnK{>U zJLiKY5$G3Q!ad{G@*l-*$^%l*#A@a3ueDj7N08aR)pFe1H3H`y#%cfeN86xRaUOP2PkW zi$mHZvB(PONjybHQUbgnzrl0!l$cA60r|SUsi+a#F2i;6Wyg-V{RtZiZb~^==ux}{ z%#_*%52w{h?_Fqhe9P#tiA-f6+%})!kNgXj>}p!8v8*A(WMYikdFJ-E?y=1h zIwp)M*e$7U{HU0+vHJ>)NwpN+oIF3_Q&fhdyR(Y3JJQAt=HdD_nod$Y=LyY&e))1B z-*Xj7+D}|%^OJHK=d{gDb`^DZ_Vn-sJ?njae9!%S@K$7pv<-f`QEFY(POlOBXj<#R z$!rvBW(Qz=HemkH*g^XDQ5obzRPRPAEu}9w<;L(+xrE5oU|jp-!V#Ra)G&;?dR6}YQVU)(9zgM#G%u|_$Mu0luRchs3LQKj@V4Sfug*m!D_ zo-=1S%f~H`1uZ3EXlzb&!RQTf&yu#JnbQ&r-fr*9xVi_D_5pUD0vb9(G)eo*9*Gw^_XG zczTY~8|>s7&g!=zXLaKV0Y~ z{@PWaRo)URs+@PACLTi1WXfr(;@5hn4xI>fR~2#}`onM3B8o}M6YBD(I6K!r(l8tw zVuO1Esloc-_Dv3!1zT`w@Jz5(C^wWJws9nX9~!L|;t;5Gf>3|IlKR8d*%0Gk5$530 zgUH0z4gl z9Uj3?6e@}_@GE^&0_blRfwE&8Q54B?8_}!zf~>eW+JJ;103h0@Ma_`9E<#oH@Ki+5(k62ITM-;2rM& z=r0?b7wif11ytx~L!0s*UE*{N?qAro6;b)-&pn?t#Rt(M#_6`*QzxQbPX*fUpA##vAz^@g~iHERH zx()u&-^$-`GUt=msoPXn>UVG&Nj8KN;Ue8M!)L=5vt*rT+vE5VRXOHFOuy)g(a)lv z#x{x{9yc!593w?li#qL?X}gWg_D6uX8}gJj)6HXGdQXf2{~5&>7hZmm!VO`H*nJ18sw@$%H(+ zamo|qOuvz2v9efFXwEO-4n*Q3E5jqhy~F*(UBhF-L&H*d8Yo`>aKEDm^+s4B7L`1p zDK>{j!-GEcL9z?<)+6AOn@mrk&(KThB&G+`S~CW7p^v5Xg$=6lnW?wgW_fB!uo3pB z_DA-Q_Ii#2jwwlqOvgw}@YnR#r^<4J-=1R9|W) zEs$9GZzN{#Ql2Ztprw2XXYw!q`6q%RrT@F;?}O7Z4<4s_@ZXMxi)bdjikYLCuHDP- z*R2Dsb&WB{G{@qxF0q;I-R!sRvb`FfvT8qRZ)X43w%^*pLYSW$UmJ=TT)HvrAyj-S zQdf~o^-bO&jT5(l_+Fm-5`K;=iVEpNxwyj1m_<{9I|B6r)sby~2lI5BV0(~E?+43< zj)%Hot`Wf7O!^zs5oD3CuKQawUSu?J-I&zYyBZbR|uFyMX zp=Pc2BP(LJdfl+wc*T@&UT)D_XILZFz368|*_Ydlwi(v$R-Lu1B>`8w&@fb=u6w7Q zqiMjDq;t^gf1wmc;>kwg9bX1xZWlT@YN!Kvw1ZK(>>XSWqVlw$8dx4!8u)--)ibP@ z$1&sd1!4MBn8S>d&%Ho(=sq$@tBdo*Z(>tCWj#`OYN7M=FSMWwK!Kp-j`Aqv#&3|P zBJ*j0d`PYgt-=aucNEM7DR{PL;SC>#g!*5p$H*)MH|I6_7Zs2k_$xV6gv#vz{>*|A z?Pn%y;z5O($M(`y)%P=;G$xvhSf*MgShiV;TN7>hHpV{FcHLUR`qI+UGSocE)XzB3 zFi`Jj3u-$uJkrf{sM4!7FlDg zH!VdicJm|fd-aCZ`X#z++EvUtDxD0f7m$#-LM$a{_!p7cAVRi7pY>hfZeUEn9=Pb= z=|AD;aANr!_&vBRcsKYdXbTMu<%gDq6C!&emAGqM2j0zZ5ju#;n1zPo#N7-Usx;`8 z#$tqC!x^<6v>gQ0H(M~nUx2Q!5ENfSp}!~sr9?Gi3;f>|a7DGC-!$X8Y&ieaK=QE% ziqu|kxc90?Or8lJI>bW`;w4R%utv(Yl$^2Soh zy2hGim90MOP3s8je-@wltZ4vt-9`1Wx|`Y}ny&N|@{np*wjymJMtH^Tj&u%N5%OFk z@X3E0T;AvY-TqFal!W!*S|11{&;qCw8=wLZfhz71IvH`$_LYWW z=L`PdUGNJ$(5V&1o3H%;&!559$wSkp1NFj;p65Pv30tCG&e1oSNBDqh1IKG5INU(=s~z5Zi=19aHy1P`IB^)NIr925B`QUUpyyU}f&f;sF5 zk{7kmU(dt2{5H`RM3_W$pL`0}65`<2@1;$5xB@WyTN^u~YuoArezf#AmfEBO;JS7$hoF}M-(CBYL z7w-{HY`yeV45f@|ruU`>;2Cc=KQL#R{pMWrF{JR^G_j^a#zKZ@J%JTl)!bv6&}&FD zaZ;Iu6v&V;g)hSuj5NmX@Nh6EFd|SXP%%(45Fdy`6=GW;3Tu}&v=PMoY~)-t;|Tr% zzaK=@u@czpIH#|I2AhY^sXOvrQt8e}(7a1ur6<7UZlq;6S7%ceP?6H1u^Ep_|9L1( zo2dd+X~mQmP==p}B5$hP7H9rqa#^GpH$@uZILv`3pqh0ezkUxmAqlwZq3|rfgmTgX z3eFE=31);G>L9&{>8+`ym9^K{e|3BGYYokecZ@Nnfu_5rXmbg?CRa@TOn&1C;{`)D zn6Lk_&$W9se=;n+lq^DARGP}~K!i)>Y0ela8txEU8`K9c;ffF9Ee{6j;UrKebRTs! zDsm~ZoEyg17AWzixJ>FRmq)+-5z-i|fo@@wbMxG|Gg}-C00#~$pk@%Y&M@^>ZFe^11v`5(2I>w+H9vdf^^yUrb zf|hxfLzdl^k(M%+H=w%DF)c8THuTU}*SWO)G`H!J)N$03Zp+O;9^cLH;ciAE=r>*u zjl*m@CA=ye47ZJ(iPYroVK;dkuEbkVUsQ#jWhhkrE1<$Z3_a;ttiyYe zplkv=s3=tjZ~6$TKLuJXp8F5Dm%Cvtj)kIeAIAD~D9;(tvHHN7u}^M<4C4(_V@V_3 z5Lb&M#HwOMIERz&WMP)DR@fmN7S0R#LK13hL&f7*LxxCR=`XnyYGE?kA0zCtLe<7 zp$B`Jn5Y(2vZZ_CT_McZ;a74T2<$@mM0jC%1XA1jho|5kT!M?GbYyvikId#A$jfUi zgoP90I0?HH`7)F+HK14+Ln`C~N{fBvkHol(k*^Ue*OJ=?!ek5nB3~Exy|H** ztS@1QEkBjVL5GqAhF4jfbqPjcBn`#N@P(bzFE>mxb~f3~2cehx&$1F(IlXNQZ4+&M zZDnl_t;OJlYisUgs%(_?Gj*@ETXhnKi?dLHU9YT!I;tmG1GIs?=*eyXy|pa*+*33o zHTB^d*}-5xK`+N#FqFDZI>-$477i<);05R|9TO?BukfD#6MDI`++^etr*IZdf<=em z99$``H>h+JTtkO=gV0}iCRBs-$tg|7$QUf=V;rRaucotrv#R>u{ypd1jwuEhx?$)T zx{*doKtd3dZWJjIBoq;Z9|8hW0@5HTNT-s5ba&Tu+&*>R@45b8&S&lnH_qL$_S$Q$ zz1H)9UsMV@>qh>%kt_lk9|2U>sh7WqUlAejfZQ*f4Y$l)kIsA#y=jSXPoR6V#UK&A|}ZXnK!Yd@@)V``(r>tV3+c_AL7| z$7APy#$l_wqUWrKn#h>lSbo2Ngq{)eCT3BLBW9Mz=J_SMFZdkS?B{G(t$Qu+oA-+; z!h^^PeTbH%9+Jn0KMNfWCI=S=UikasP3QL)_1paEApRHk5B48{g`q}ZN+2z;IXFGk zpIo}XS-S^xddZu_f zd3t+}c{MIo#L{wb~eViqh(GrJvG5 zsifq}-^q35QgVH{t=y8|#pT}ePPu?GNx2D^(`L06@r(|{&>Yl%{f=$9(j>#aJD)5@ zUF>MyVp(M^ZCh#^W?%1E=ge{b>)ZyW=N?x<_aj$dm*y)%@ zb;kw!L0dsG95O9?%vNatS=;%^9dl@hl~MBO@ZHcXa7QOo>9;JfHINlZ3`)UkfnC^z z^@6j4>w;ThMy((6h4#bjwo8szHY+344>S*$iEFSNCSu=xMH^Rv=LHi0^;^HejWvP1 zugg@U$si<`6z+g&wG=CG5||;qz>^g8BU%ca=|k18@s;<2Ynr5H5$$ZGR#rpE#c3sn zI>;Z?-(U}Qv1>lm67^bCn{}lcXDB&Wcd09CPNsV!6Ucnh4)X-dZOc!DR=;;Nb0#~l zJK~)0oEuy_TxDHHoVT5;T_3p@yUW6T(F)6=w)?(ole33ow|$8Hx^27ljX6nLFHE7b zWP)A<|8R`MS+G_x1}=|;pgVY<=uC&e%)oQ(#_YiDKqN2#1c=U|^C5S* zO?V#}K11>CC3gQiWFdY_{Y)}Z>X)z>*F~G;aVlG1yh;uELg_PXs#wW}ZE=_^pkvsQ zm$4)pnD&6CHwAktOavlb7Yv}rCQL?lXVF5QRGy`|HP-fx{jsCJvy?N-G1OVXwcIt*WpW*GUWN1Ig{!># zs%xNYrb~2pb3b&wbmns|cErHg_|E!~Wk2k;%S;m^RXNkklYa^C4dn!@2j>K=fffEX z{yP3a{<;2s{!;!5elm0YZ`n_(21W!<1nLGK1*fofS`i63CeKxhs|(c#XVb}?e2t~< z$3ykeU#3Bz5LU;+=p}w79utGe^gr0x`N6`yip*%#24!F!*NI%F&hRxjZpm6a$fOt5 zdFo*GV`_T8<)=3`R(EwiW1XZ`)tYgFxF3X>npA4lBm4Yku$-1)W3Gp_To!*ehuBq4 z+pgP-JBzv2xURZl-JiRUy6?F6xktE*xFasByMX(tYp^TFdD{7#v#;|nN2a}r{cGEP z>q*N}b1N#b-#4lHDeZH$q>>hX6p92t2%ZZJgW=_Yf31Hd-b9W+;5Rc0U&0la8K@W> zg$*DF2Vim4f=BNS@$Ha2fvVANR7FdN5$Vjupw$mvx(+C zluk?AV6JdT>qPLb1V28ON~K4E$SioUPto6EjL2MdH<{eu$;tAnaINsd(1YOBV8_s+ zPz1E#0`fy}klRo(Vu8K*58;T=f!h90Ef)JS=S%nV*wYC|^Os5br|_@E`j?0;8C&B0 z;@gUvihQ2bIPr1JWk-^wyJ@IWB=~3E(X6WJ^;0juYw@n)yS?w)rd7u$h_b&gzGejWZi z{7-0Rs1&>4Lh@`%Q)gDxRGWOsy3$aorgT>PpIAklCj3pEt4~j)a$p)Wyhu${i>RHk zGV7`h)TTtv#;GS&o%qreq~InMrU5E5e^qA^H(ZH*=>fOQs_oNefYSB{dFTnOzc=DF z=_j*nu54LOerO%*9gEXa%A6wA6~Cv}vye%l8lVy90j14Dtt0Ij&O$N$;-@7REjTI3 znbf<$v_xyZt8tgyO>A?7nQEKRO5f<*D_P%V6i;uOo}JMi>;Z+h0M^75 z@*>B>CKD14!RLNdYGfWoj^#+{XX*pblP}mzzpHLgPQfvnC?6*S+8rJet`C3P60o;E zBqrr%AFV>A$4!&Vz?7k`G=n<%Db&V$s3bY3T~yC0MU|&=u3QzK=1NHA0&S~)7i6I8 z!W!`l$zrZ=?qvSnywIFv{s4xpXT&g4U>7G>kag+jw6h^+j5F+)oI^Y<;?fd6%HOi! zUj@e$uqMup9~wK)UBdoKe639iAM~HeJC`#vyJOa)%r{xxa`8d~)xt$pGZ~$G#gozy zb64|^;9#%DlUt+@U>0*|%O~o0YJY8kz8UT5A%>DpR@pH!K4wNr>i1PaU4SPsS^0yL zu{R)+E{fQNTB2RLAk8tqHQ%S*_)|#GNK>1Lg{<%WN+OlI_pv*65xLu-d`%8XCKy21 z^r2Xulc=d&Mhr%wmTfh;)PvycT(1^T*DDg|5OA=DSA<>iQF(-7R&T2rRI5zKB1$3K zwFfKphO|c-D zWOr( zQTivdaWKGoH5$!BbM5{6Q zt%bx>;z$W*OLGVF3cQL~b89k5@8U@|H*Ml>Yq5T}2#?GgZGSt`T#nct33n1t6zE;> zZT_r;(Xl_fS2(ttm+7}c@xC!Re`mHNuk!_&onK~E&ubj`CA?In(n>sO-fl^=Y_5rTGMP8%M=^!f?}2eV2NOy)~7}w=R&nY^+M}Ijl&g)lEteZYgcs< z>CuQE%)%pBLlpZ*qF(KZ1xa|vP4s`LclUtC`~!H*&B;<~4Q9hvARC0V96axGjMiLY z;URq!m4t?@fyzl+cUS+eMDrn{#)OMt>ejAnEhDvs0G~bdkiFhm(HkgV=N@%l{zu8NM zDH>;5ziKIwBIwZx<^^cbq1JiUUe>u551DJEm3Qwe?V4NaeJ;a$8$G+UjFZr zt{3i5C6QmUsIza$~a(Wd54jI4dptN#5UqW3pd!2u;lMtut*6 zY!$7S%&K^U+QfS5)o_@~DJy5?S~w_w!pZSE(zYInz4QdQQ_Dw(n_~zoT4dXspmOJwBmd8(6MkA`HHL{d;O@c`dhK7L)49oSV#2$ z+VVE5?=-rwFHt;QXeSjQX7Pn2Npr;`;Qj53{H^cj1opegCsI$_DaPk8tRs;^p`wq9 zcPu`;$n=7G!nK$`9M8m9^|pUyjydyETI;m(>2ov6XP5V$53W)Vn>w3+w0-B09hDq2 zY(H32J$SYgu;x(v0H>VNfItBkO!l3v0j{ zSW(%l9?(s~B5AutwFbzB{=~kCjN8}E4Ia5%Qx zJk}>tFJYa&S&0vC4W0{Xq1K!x{HCtfV}#G8Sj#VDHMBq%mJz3#rt2Qn7rq&49U1}e z^3m`R<&f6c^jJ(akF%b&b+vzFKVU0s70exk&H6Za%k*HCfX819Mw&kYu~=cXsieBD ze{WihA9{rIq?*F_rjI$teX5>T4-yy=}P(3-4IbDkMgxscQHm*gq=b ze|C@yW7EDe{UX7HXODC4bslmab2#mshKg$=AE_0?qXH#;qW5546Ypi;#=x0SA!WMO z&a^;$0Upvquu$59q`1!XR`;s)mFwV~?F@Y$Iu~jP55)wS?e?l~wHx{g>f~aC=HRJ3 zVLWS70bUx!>yN2_>`8252`D_%xLT79j$cDmx4l+5@~3pt{>t4k?%R9|3eG4pvG}Nx zo|4|8rIW1rK98B=_*mMg_6k(a?UGp~{dwBO^p7*Y$$psECHR+eDbh#kYF%ir?U-VJ zW4!`rTX)kE?X>I-M*OF#y_pbF!xm)+HsxWVg1H@ReXGn9q<&&P>Ps8(zSqP$GlMHb z>%tY34parr6Z={Y*+x1(cFu)&XRW=1^((1|shpN1mkX}(J<7|-b>yYyHTD$=bPZ+5 zTQr-gfH$olb|H?LD4DP(5tCa!hZxW2bKg< zg6%^2<<;uI$WHMm%kQ>Pj+xG~u3Tp;$8qam>1brO+9NzIVC3U>%=tEFXYMI)n*Y6! zi}m-dNs>mGU6zN~Et{myoR>c%FQPP7fGvC=)F#v{bei+;n{o-Yxz+~nwjQ|BFG1>m z4$|`+_Ww*yxSPSL&=1V(uc;br!?-=jTJKUD8%1B8dxhVGJa>G{G1Cly%Nb3=UCs`ySs{e+I#AHR=M`t|7ZD9xUPLFj|xol*3EsIT^W3~XL%p^=LRRr zv$PJT8)AKPwz-LAyZMy#g*edETCb|UFUz45;E?X(gymJRVR*eCN}*pCqwbnn$}kBnjdtW8z2S!qW>k1x97xk$#=29^&;C!W|X~9VN?0L zWhRywnA9vG)nzqbRC9fvtR`vC-fVcC_2$64ff>hh`un>m-pA@-71bV|7p2ysRc*%8S#RtlfFm!Cj4jcdzgI}`AY<<1XqQo$Q9Ly7K$7eoYE;y zm%flhX(I8CjlxD#9y~F%6<>III2h7{LU2&vf1yA14%XP1tBI8h7b$hFLYXS7tCy_q zu3W5~zet6AA?G{OSHZNba&No5$p3Wylc~?wzdn_AAg5aJv35qfV}I+O2cFpyPgm!3 z>neDz>ndY{vbRreo!q+KPJyvut9ns?DNMBNwtwf8T!S6y*8S$q!oT`3rA=^^uK^5R zMf2u)mju3%>*<{&pRJs`ZQO)}Kk~KCcQ)=@&whJD>5z6X*gtP$W=?98cmB6ur+%1m zI@|6W9dc;j3kv>DcYCI7wzawWq>!pVR}O}*`Sbfe@^17_@-6T;pjshUnWV2W4Ht(> zPpR?UDQ%Wsi0yF4d@2t9SL3 zO2bPxEVMpuly#wUHuvVc?l1B^+5Yf(rcnht>YRMHLq4d*tNcSzm&8 zUtUk|FFs>mKVjN$+3JY%w5D3ik#B24k+{+Bu{MubPa6|@=e?EPH)D8O%e2Mmwk%)H zBX4$~wme+>z|@*NzHg<{oJkx9Q~d&!mer|b-xs*;m;HAF-QZ@tCy&K@>td=UPLk@F z9p**ib<<^ijk;PsA3W@9nYSSKfxn?%-F83rVS$y!KS+L3eR!R6^@DZXH76&#i$6^) z@0_a7_Lfim`g!)lS@#;>zwqeH^ZRdCXLSoa*Xo%6aZHYx7JnqZLF^D$cXBc}M}AR$ zLJ}_I{+henyEX8AxQ)77KPJRjhS)kfE;&q&c>7jMhWME&SM4Szh4%X+zQf?|4G*1C zoK^Ce1A%?6gl_(wVl+=#fask{ZeIVnQ5)oMVTBb z6zJ})l6%rWTd!~5n9#mZddWMLl4`!GyRyN;x<_hWtWc(C$$XRTo3-9v{oVEFTOVz@ zKlXv^$*7luQ`hE<2z82Vw6t>m9J3_hcEXo&1>I|{_r(*D=IX{!VPAusPqW+QPVgNI zE)FLs_4HqaZRQHL6}FkSY!KJK5?!W$)J}58&?hqKz>Z|oT*YQ*A=C^O3c79O#-mLo@AD4Xj_}$Iyi-8JS zZE25vL`;)}K?%umE8O2WTG|#6lg(2l zAK2veU><9Pe0qZ`f@^~V!|BQwrlywF%=t}ss;78t`Pk25Zn#F-)`@#Gr(7vm-Zw4x zLRRt23YmK5(d-{`?Owm{U?3$lD_l>mql||QvV>j~z@8X1RM5ueHLNLcy(_1LGS11!B73yEIO8LQ6SEMYfx2M60dM9g!DqJhlIR30n z)dYXF%;s-Dc~R!c&PUUq1YXR1S2nAiuS3|aRTFAhJ?_h~vtu8)yE&#>H=7TL`AsF% zxxo*73%&PzM*_*Aebl>+(r=mmg%RK{%V_ICiz?m3&hminxGQ$?Lrp)oA#MpqPr`pK-M|zvr`JSzNWP9|}ldUg=cTKWR zdk2JmR1XSM?VDo$ir*bSCiaLs;{3z$vn@@UqYn#D2$c0dC*$F;uSDR((690#wVM7c za$G1a-Ix9mr+^gny;7E17lkv=&B}B&S8pLUx8AkibGV($9XXD9PR%jSe$!HfI+vKp zB{fEF6b$%Ed*|k!&+eObJF`SqRqC{g<-VWS3I^9irIlV5Ca&J(Z*7GACkD&9sb*K! zgbReK1y=^LeIJvz+rum6?Isp6SO42pD9Kf_Nx8n2o7bFNcTR(r^^2zrt+>3{u6(DQ z--*A-*YmP67Q7wtV)WBFZvSe51wmrZ^853=;`YD&3()H zxAm6kg0eAW4VDjx{w2P>{uaUP&!L_HWg{$NOdOcJNK(vKMDJ&Z(9=F7JXb zCUjnD6j>lx;WBTJe_d8uBOHo+q`i|j!p&gwne$?DwVZu9L$cGd26&t572I2r-jy6$ zen;hgHEP!$TX%n*uWMGT_`3Ls{QEswmSnwQa9ZxE^fj;Jo~e({Jj!}f_GSKe&(lB3 znwonv*hI*7l#NY^9~?g}{zd$W_=R!RJlVE%@wL8PIUZ^mXyf<$p9Ly&QkNFk5&S4D zDR-16%Es{NP?vB4Wx2LBqMEXVePUisc-hd4(;{5A8aB&SGz|#gYmpd>P_^#t)-uNq_`Vo z+S=iSz%p;k+@4wM(s!oU&wHY0yN@LGEP1`$@XGmXTuj+oC%JY)^)cml6>XHb$34pO zl|DI?nRhiqeB*n%?a?=n?jh5i-kf@;rPa+o8;CKTwAFTZj9D0aCr*uTn2;3zZH(VJ z#@5t)!PHZGDrbg^fI|@%dhY+k*UI;S|Lef%K$ias*?XG<-tc&>lxYXO?Py+Wt?ZCo zv#E$1;;iaeL{&%wsjqM>Qb^a8NpQuULa{jJGou!h> z->tNw+Ugo7tG}omE?cTty8O=6dS=ltaPqd^veBUNguk zvTw2P4`1zoCtO8cO0N4Z@bDtSc=KHALE9GlMaLzl;A-Z~v(L2Mu}qMXOfQsQgFpKU zd6(st$Qul*wU9e4w^`l*@1MSx{=Wj_gX2Rh!e7Zf@a^hzikJt}ffcOFtHLu;lJ*Nf zMTTfI!KBz8Sm+-d9wYYf#1$A)q;ZM9W$INhR~l0JPNioRzAZhhXs3c_67GAJ*ngAq zMZOM~&HFMMyJG1-D|5P)#asNw==CS9NZ6**u<@ zF)=yh^F444avrjMD&2%*bBwQQUh~`^a{kPAXBW=?Ij56%VqkE1oN}EC`%kbcE}3Up zeYOP0bVoi%aeJOM&2o}V*Eyz@T83Pc>ceiqNr6xO4SWT?g{a@!oaG78h*(qD5K*;dN(!u$$Mh5S_HQn^W``W1az(4M$6_8;dH3wU(q+b6#|Vw z6}mvQvLpF)C&XUnU#(Z{8=XVl4Py?+j*D9f&-)@zz@8+X(`@pB-~#`4@8R6Pv%kz* z17q{gd7t>c_Me7n^Mq0s{HSe~pskf-i6fD$zdx-TEoaRKq!h8K=?lG#R!l8Sopy!L zeZR@uH0RT-jLhL#1+&j&ug(eQuJV@h{}`wNg3L2wAcu6pG)MRlhLoii!2(Y?5%8b& zhpI^%B6M)pNnBRsV)0)~cuQ6+)3scDd1tvPrRNrJQFvQ{1&Q_Z#l#o(47Pbqg8WTh zlk|nJBQLUElzTb$#nk6hUL?LLmi{!mqW5EeC@?#;L0+LX6Rwzd*bX~}x!QQFAg~{e zYY=xgW~uutM~tzrv(gb0K)XcKl zdf%32f6p=BUedPBGTHn;slB*{9H%#G0p&pGg};M$6Lpv4avo&2$xh0im~G3+$r+RP zqc1gZC)82irL5E>Q!!DHW{QmjcqR3QaLH^SqrwzG-Yr_bbOM`!xGpTQ$oa z;bXm_vOV-&;IwaocYfYYR^zMO6mKK{05GIW5}P}xj3T!>p7Tx+ZHQoA*8c?^?|BqK zwQS^PeY5sI)u&V@->`ssQ|=hPgFW{wJXpRhFH!2K^Eioh6B$2GeZXqzj?|5+$AeTh z#oOB33fg-)PCHAv6Fplz{m7ZVVU5KD`)wqbRRrmrNAR94wh^e+BXuGt8;n-a!967yS7Bz3mS2VeDvL57AKN> z{si=#ognGmHU+^c?LxNwaG?)qy0Gnn09;9GOFq|2ahX^b?$JeH3Qr&x+XoijZ$xjt zp;ya6#XApzXdd`~Q%p(1L7|@*<{axT+_n?Vo2kTGVr~W#z!|BNw4Ml92FQZzA{8SK z$xLa-3H@*w_41QX*q!Y6WYq zkGbCz=Jgyov86bH7zQr$eHhk0gm+I6CUSn$RGb7dARHQL;C~8U~?&`&W=0~UXyuNR+tTL z-Wsqmp9y`zZ9XMc6&IPT%)=9~4lBXvwv0-H+hUyb85wG8L?0{SFcHFw`UlEjP(c0) zJ`3Me??&EGZ{Ap6ty~Mwlatjw`Xgb2WRmU)sSyDbxT1O-XZ58x3wsD!;6K_x@+2+7 zW^oMYKYPW;LLzxy5*U!L$>DBoUPg9g1yHG*ksJP-@RRsfEH8DH-k1KA+L;G(f?v+; zkk$%UIF-&(pR2>QF600oid+F%aS_O83K_=-#VtZZlPi)2gS-xU+dsjhf%bt_kf( zKz}S0d7p@GDa{0$%u=b3)oyD9n)(B4Yuh_pKgV>}QukB0#a+;4C9mj?qlaCyzO}qH zyUpvx{mj5HTJ}G1F8yk0c~G!Fyvv(?$soz>2s{gx3-{sl>#nj$EzY^rYi)}@m1^Rb zrlDklcHz8ux_Pd7hWRtHZ~q|Qu%+2*nPE9>zAu&*ZoucKf-SuY?a+hVa_UaB?&M8W zBF|b^N2s4D1?A4+3E^L<`rM=3luw6`!7+0xbTjly-k^4(rmVF#NKIDrID0#%G}IQTw5NgCrz~tayE6nb*^=GcCK&+T_-*9 zaq|;QiQN+yCcaFZmAEutW_$`fElcc;EzQL2$P9fhHC79hDsn<-jX%kIKW9hw*zC92 zf5D^@WD~^VarS|4_pRKt4 zYx`t-jD4@|XWLJj4@xvSL$Vdo|1MtAIkoD*H|k*!tMGupG-bI!BOQ_?fkJ-}7ex!NAI zHiIqjsE}Y9qes*Socvqm4&nZxgy1rNXjGV+zG=_S|*nx|+NGac*|5boO*Ua=ayH?WARe z^nqZByi%(u$HPB_W(HRU@<7UL29n)!@YjUk>)_!~pYV-vS(J<(gppJ{JSHp36?qE#?@g*UAIc}?OY$>vcW=p$WETkc*OfkMCC;5kf-Czq z*nYLi_WVI;DI5S}x1jZa;~URDBxGd1qY z7woIeO@!acLFu5V;Z-366|;i(QqF{|IT`iSXQ%Z_`ys7h`m=Oj#-J=I=Ur}=H`D(z zcpYSic)gKnoH&zQty=g#bzJYe{q91Z{qB73p03}WZJaLWHOCi@EPF+JVcS*ssOpGr zQ=V2tJt{vWL!mG^Xx&18Qg!&4v!Dl4>t*~Mw<--BX143@HN0X-$E{Zy!Z<_aaE+C zI2-&!Cpm-%;Y0nx)JiO5nQMRJ+8;A2?qOWJxE8U!Vq|b$2jlq`PB@UzBjH$Fx~H5o z!}1uLali5^e3q($zQHMhUj9qoK(3lyG3!}I-;8Y;zhoB8K9%zYNPWlrGlQMO?}O$x zDY8}ABh9nqv$e9Rx`JwBjtDZa7Jtqire5Ux z8a&-!V&lxAHb0IPG(|h4dX)ysFLGnqEhiyodpP&sMn33TwFG#o!}ZFr0KPV@73yM9 zJQG7?J}wgrh@&{$zlQy`5%i3w;!?{H`#t9xDxqRM6Wljkf4Mrl|MZ-T&5r*iU&qAn z^L-S5!^3&IdB15CJS}&^eW@|1AKnsL84UV^-rKnu*{ia&%($$hSr3T&{+-vX$(yidm9_CCw5N<@yLcG#RX|6a}TW94|xiyTHjnob5T{TQjPIGL+spL}S z!Ap5xa#(I!T3GwRSv1wMhD^XR;(72PTVSPWrb^NUqO~5^Y4;@02oQ*xyPvtzT?gIu zV%EoAiA#zHe<>z~`0ZjT*3?IvpbVv6B`drg6u&LOJ^t$6#W@YLPG)q>_$Q-qR!(*} z_olbEe;M`csiB7QQzfKzHJuYR^LcAEdp^fnht=82ImY>|v!659(ZbP&^_9*!smDB# z`j~IUG;AU>^>~A|I5iba#3|wY;qye8T2d8~92$uBof7UK*Hjv)#i+bF3cJ87@P?0) zwGtzg5^7;Fd`!LQG0^cViB+j!T}5`}=S0bWjtr)5E|W|uADKgU$rdW1<|x;cW6E}A zE3$Q#9HMRNNUghGnhL|Oz}_!PrA|*U_V);8aWt0idDt#AYKwiK@BFGYCCC25$e+Rv z^I6*nXD9bS&jHU*o?4z%cbw<2=UU90*d=kx<2J<3_e^u1w7wPxM&_v-<*meX$AiUi zA#^_2DliTPv6S4&IX~pI$o(wuop-o_mKppzw2`WklS&osnGVN+SlaxdrMPv0^$+U~ z>vHQf>i}yrYh|m4{L_(^2j(hp0sbid#p;=93WI?3l8V}G8punWq0c6gTtz8K#I_XK znstz`zm-ow>}U_`z#-EW5GspH-Nn894i(BXVs`sX$-RWArLkz zX~Wfjlp4xbxs$9?`FA;-iN?xcpPiyM(z3P5)HcLLlF0xmjmIZ}QuLH74@9vRb2XOR*+3((!bZ%lSTO?;udF{6Kx$FH=Wm9_1&YW-P+`C;z^15D)wsJ zn)nCtmEwoRo^@NHFww+XYbTPZBfr_IF z>N#2WvCMFJY_Drn6^v1?$-QJzeifFeL%A(~iAVaHoOB=l;#Am%o{)t-5RX&E2JFFk z{Rc1`#A5AV;ku0nbb(y-F8IOc@X0@>f-{C{&ItLnB{|U_PhQb&kg=QKMeTq&ryD+P z3HZaelQ*&$MwMe^j*O?`p$9qr2RI9vr7ZvrV7~T+M)Z)p_D0$`yrRO`uye@~G>Ij9ME5LJW2 z0@M9{d`I(Is z&#DIBOm}jyiZG&P>LMPKQ~MKjVji&sS!;)>s#mDF8AUC*V7fwOdJmYTmXe*jjP-t; zdWo^*SH4u+u)bHotgwrU{9I76w@{luAD)bQ@WampX<)wi90r4X?7-E(!=OBYhQ&BAunL zEW2$-9i3dW+$}w`JYRe2c@jKTIkj2gx$jAgiFlfLGF(2#54Kn4IB|YtoOV|EQQjFY z8eS0E7u*u);ZFnoaCUAmCy+BPcVOOrZ)5+3K=Dv(a+6c#z2Nxt*Z!yb!3%yS)-(6D z6tyn0p0S<-FJP&42wABk!0YXA6|J)@Yt1XAMP&D1Alh*YWcy8Ic#ndQY?JOJf3Sh( zg3V}@c18P;Oy?R@aj%AF{W7BzhWn=u=%(*c!S$AW9y{pc8g|PvSb%nDPpN15f^jw& zIdWjX^3nINsQ5jvR@BzAnx|0N5sxnI!*^|oBvQLk4IJpnqD6X7>Oth`A8{xZwWGkm z*#lC*an(p~0=6dYZxF+$BLDt`#Fg2l6!r=I<*orY{U1c3Btk0x*FuknALp&u9 zr#}9dKx6+|@4&o1xud{yyO6D9@5-5;yD;yRH|T30_&F$tMzVX=#sUsRdI{f&$EeKT zZJBHJ*rtLbd%<=U544W$Z;+5br<&}!R6&{~=7QW&P`GNEVX6pEPet(Z6ZAP+N$nr? zp!zT9FI9-s9i9;U ztnTkPL4HJD@CW1!)n;Yyq~>Qad|Ladak#^Yg<|?2NJ3jgC){1Xl98`U68RMQsa&XT zu4b-7JgO%Dy9ZKEQRz$g0t*Tsp?8ua8(^eBtAX&B_ zP#FH^Are-SBSp&iAhJKKD-cjU-0@g44TOat)BF zKE*D+0Iu#jQwBAtn~5xaAwB>bVV3lpbeL*`_5AgbR87*UKKPZ&fM(c7r-XSRrc{9& z>M_+GxmZ0P@mw;mOHvnU39M!tKnSV>#`*umiefs}(6{J~D#Rm~Q?2k1Qu>5=Wkmap zT>j$POZ9iSy;`d!)m-I!7>g$V9Q*dR zAc9^}4-PXAJRPI40?7?Pu>pR!@3615e`?ok zvu#0ZC+i4M3rk2r;Vw1$@9VX-p6ZY6Rc85<@U>9m&}QoR4g^+`rIs4#AB+p-hW-sN zkUQa%;<^@i9I$H^HS01EgUqONUC<4xaEzjgrbrBwLQ;)x;0!A_V>gzPD3Qs+S9 zxQzbZ53AQnY7pjwm!8O(;kTeN6oOAh)gS0Lu}z-AH51@mFM&1L9`uDZ@CiM`QYi*o zS}*w6#=#`^1@Z4bux}Ls-}@?Bav!*UG2oJHFkOe|MMO{KneKx`^a5t~F+w(Y1?lig zeum{~qL#xVP8PCQ_vPWkc}~py4&U$#He*95TL&#y%M&mAh0D zo<-u{Fgjt{wt*3N2U1i#w%=ed*{U+aA7CLIgT=5Bns_cz=Sx%%l!V(Po)t2gx+=f) zyLlw2=d-Pkt;IPf8DRU=Hs1D`tp(VS-&%7m%`9WgKY$>;lKAhA$Pv)boLX-Ae^KS$IlO~&>J4&Z<)yMbTBBJT?!0@PO0;FqO@o#0xTzLC zO%_u29_J4+R6YJn?Z^gJ`#$Pr^WZ8h&njw2jNK;sgq4!P08SEN(%X{^cy@CKWN zwXpB8n;mr=O&vABBmdbRYyZ^tjddlS?HXwld@VQF6AHt1Fpbz#qWY4Lhha*41Y8oGFTSQW@^G-x$gE zSpIgr9}`$tc2;n85MDZA!4Jj8T*P?ZLynsuZ}VCCo2fvUi9GkkgZL1Ry~+3zOQ_>q zz?s@$n7B$rVz4dB!mslU_Lx6Xfs=y)tjYnLi6yfh9iYYKhtIGJSjYcxmX!n}UR$9} z6oaQV^?ZZiTiA}&WT1OJti*Rn$2Ok(l)k=4bzw8s$_#iD4S(b$*7#8*ZN0FR=Z0e4 zHGpf;hL5-opGSgUrXQBk1(@Onal&UIPj|C687!-YS~cn~8}iqeST`qG!9Gpk9I71a zdIVO`1(4u-gX;2_>fUl#F_lae&@M&cktv4N)rhtaW1k<6*H|9^_9T7p7io%@8As1^ zIOV)Rg~4J@#oEBxn}By!h#KBeoa70d4qf9s;-IM^=Wn&~-anOMVe0(M;)Q`;fiP+QnK!>WL9y5B6C8%cx3pRlaXMjqq$CTMNcd5SF0mmA4~e3u%Ub>Su9 zMex|J3-1SsAxQ4lm-2IPId&-y>S@ly!!)10s1GNvm8cbYO$%0_{oI_iOd!g2S%|Q= z_5t@`E|Ri{c*j^6G&^v*))sjhO#S~C$j=8vF5Kc9DxS_#`E-}n`&zg!oM)AfW$w&G z+?L_Xv|)6rV__yS&zb0^)l{q0!q&b`@3*m6EN67*BU9h&)3GT>Q#aoR?wZPaMJmWr z_`53}Xdk^VKi$C?_z0~xRbL5;Tt4*ZW^`sFtjhIh$-A(J7*{5{TN*z68`C|`M)%Q% z@usfKmK8jgRY+;Qh>0`ao!sNQsK(gB>EdJE9H|MPko2X1nDh%UW`veHNQF#_G46vCJYL`V8k`C!~_nbShST^m8aRJ8sY+F2bF9TNz2r z>##gpu0tfs${t++9E8s~n-G<5VC3a0?TF=AsI54Fr#(>rTQAQFc^s+EI=IZzD*>8T zN3Kpt@yA#{(~#+{$o5&`jIdwW#W*iur|QX$Tpj+Ys%Xn*W#F=o2LGF5`IU8jHpWYFH*V#U5727u26t#4^y>Rj$DleU5eVE4$qs z zv{rcr_FDls^wtxps2oKETZhNk1}Vs=$71uv@VglLt|BM;$$DY*UO^tK;2jO-WOpmx zXex*f26Jvoq(0iR4YpfbtjNCT&LRBtWXEX5)qs8$;oLHTh*uKd)DTRJo{_Qi`+K4R zf3Ta5@_XE6@!SJXICcm^TJ6eiRm!F2buQ{(bWNylVKa+uk{fAjF&WsJ-0IcXAlWHj&_*}17S22!@i@KEUcu`SRB=f@mFFLN<=*x zgVp;TGO(Ui*^OPoY`TxnG>6ty#J0SHhFyXs+nrcI4g958?2Z`XTctqx=m;T z+ieoMegT?nCtlYjSZoa5-pP2^yV$!G__%u$$=ri)c!=vX$R%6Q!F|C>d&`O%jV#*9 zH~sht7^dIei}=32>U}@tj3nKxfs5lh~+2{*GOUCo7#AW4oAJiZ=LdW@hs<{>jWk9pe3M8OOF* z7Il%Tvdm60vs0dTIPuS(^7;G7)>SOy>uALn%y1quoPosT665zqF-qO6omk|iFn2A- zJ<4zgW2~xi|2m9YCH&hszF7nl;0020ot9nau2;zExyR30{%2r3dXSN7_?b-^-+}C+ z(^#EfgC+SL*HT7mK97sAK}Lax*_m0X&-|C5rABW`^DTAgLkct1fgGfPpg)XdhQ7jA z{er(f!%FUsR;$G);*oS2>p7h{e~Z4mPyf#1g&v0Q`ByyNRq)Dxk6e7tDjvnE`2-B7 zkCCts(Usj9?+=K93_!0AmNLAj0%KK`Xa5_uQoK^0XR7deb^6$jtf^s)##}0IH?s>}WUt9E z`As6xgD_GT!bW?xwlLf-k<1ziRh!Q$VzqeSskR_&}51Dy>WPf z#dx+MJ407ib#LC+hhBE2pMCjz5Sd~|P4pIQxiiSpOLoOPkT?x)k$5g+1t;QhIEW_u zc_kC6$V4`>K=3rA!_Z7`_|y}0$iL{Pvo!+%tpKuz@}dA<9vMa|tgoj)UafEg%za5hp0~SWgS!xn2&E*aoV;cVTb+Lr!2C zF#sFmQ-QH*$ykgb8n6b-;6!wV{l)6K&Uie<+J8x(17J-TWW0@;OhJAdfm+m>x&8<_ z?Z@o(LmG#0jleD(#9WT1b>o@QFY$BcBGtw<7g_%T$?nVCwT()4HDl??B%#Ljq@^2PT7(Je?lT!pon={EdYlw=Y_*Cs*&NB#&da z|2pzN_NGnr=p=o4$m)~O_=(&<1?|=@x`IBTZN6Metp(U?=^BJ#{&M{fypT#CsMqGi&JoCgf^6 z*M8*eB)a7;a`%QAdO`f^CBNVC^OR?vu|8i%|Nn+{`rpq}o_$RVUh=MN_QD+gddroD zj>v>PLI?N7MNUV)=v9>Vmq+ImMqd?VhQLOPt^*OzKg_wH85=Z)XAPOCz`T`U=1TEs ztad|7)r{^jO&Fi_oJh>6Ax|`qcV@Voklcm9PxrrvHbe8?|;T=;}j%f0pq!h_pD|2U&s5m zVFPSt?>mGHonk#6V(fRYj~K7~#$S7pckoh}mz%6$Lt{T?j9zdV+CLCQ+!dR5VCWwpL8BXNR6^&XL<5&{Qp$zY+ zip#`XC=8=-~Aj-f!H~kW!=9 z63<1VJG9CQQ?LUf$TDz1v<0^u4dUQiN^&JfN3jNvhDFenG4F<)^37^MhaDr#&SBW1PI_V3hh_Qp6xwCTPTlC-mW%F}$;@y?{z_)H zjh(bu)Xp?|ZrCDDtQ<4_HO9;sISwpnwZQ+3v9j_W8-HuaaVoR+ig|m%_p6{xyZEe~ zS1qir0BupD>t99QO;KHCjG{4*X5M3GW(-{%$Jjf$;uuwhKKSSfG;{PR4H*b?2Q%L$ zfOwtBGsZ|6mf1b}eV>tgfOHx9$IwPloggoUPBcYZF3^hqbfiR!jQMx-t#L?2K}P34 z+d3iIV>_=KZE(>_m7W^w)5I*9keH&3Pm$C0YoRceJ;!c{KL^RL1jJh|hdTi$#6zXSVqrHz|R87pkaaUt? z-(h`UVMcDz^1IB6aT@cIxgfcexiBtcPK~u~%+|f=d+yP`$E z{qCYUpYYD7e9FfNXpEe(;+*u=u$7Ck?hISG3cWA*e=})l1;g{gIp)4be{;FFG4_7s z&(B&mbcl`jB>tavD8|f|U?j_Nr;lBZ!+|s-!Ki^_Vw;L%Ekr6L< z&7hA)A0F`dl)vBdj3K=~=9%PJ#@R>zgUq`Y{cd9ybRg@7pXiLb@2-L9J^#~0h88l$MdLM%=e^O_jDGw0$>uU75?Fd( zjnE=vb>;9~-sp1?qBOD5wUCc~myGt=h$$QPbp`rgo~s1&UYy>$X}cx5mSoyu?Cpjx zWBhIOz`|?BrK8ylU+KR!WoS}&R8tr-r_#1;zTJ>?wKSryJ)s(@!rSq4U z_ZaUFME{@3_ZjOam1pz#oEH6FLnfe);4>Mt<}I)Jqn|LO%~*d%+YArO&|8MhZR`fd z*yrQX7&AlPmyGVX)uTGHI=ax9^O8Ik;x)rQF=WfI7~|;&v>yC$tU`|rZDM$WGP4`x zTMT>MSoMaKx_MtLpGf50PQKZApRt=5pVVoSArS%OGKU#{7TxonBJU5QTHKJO+x+hu z`=+rQ7^~_QvhV`wdVy@bL^6!M(^$!&s5HmZ(s)`~j;k`gG4^xALrO-XOVfjbw8MzS z8LP&S?1DU3o}UuTxbd1{Me2;4F%Q0|WMpxFV;4!KS4K-S`JKh1Aw|af4E<>Me^5>H zs?4*-I0vJnuJW9*OB(Tz0If9UCog&*Ln9j_;X>!dGp;Q%hV#swazw@H=Ypm)2_Lu+m=LnuvEcc4%ZpOZnkDsKdrZ84@9QVp* zW!*=+oM#t3&MteC(SFP`##;07jmEbdvt&iLEBrr?Z^%L~J&VrHbACVKe>ps7jDzvr zhK-=pIukuJ_I5+MiZHfG{7yoa{_{JGURGv4jQzGE^HVEoX&Ec53@yt~+l)Oy;f~&@ zyyx(k#^Gh^7^rD=5yc7rO>{h|tzm5iNL z6&WkWW!RjC)!<<6jMZ%HRfg^{#wQS6dxrGeSQY=Z-;e^s`U$XV|68v{D-F%y#%RFb;K8Cc2jHNM`hU^%6SEM(_`nB?m zhQ>3rKmppFi2NH7HpBifJP;4B8e?H-?nrdJ%->N8qy4N<(&w&_2Wa zG1jON_K#fA-OwBPtH2R)SdoWeKxxQ`(jh7L9)!O*UTVBNM)goh`eN+nhAwi^H$xv8 dYfxeh8j}CtPGQJ`AyqHYh?f}44Eka<{XblQv3URh literal 0 HcmV?d00001 diff --git a/vocoder/tests/__init__.py b/vocoder/tests/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/vocoder/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/vocoder/tests/test_config.json b/vocoder/tests/test_config.json new file mode 100644 index 00000000..08acc48c --- /dev/null +++ b/vocoder/tests/test_config.json @@ -0,0 +1,24 @@ +{ + "audio":{ + "num_mels": 80, // size of the mel spec frame. + "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. + "sample_rate": 22050, // wav sample-rate. If different than the original data, it is resampled. + "frame_length_ms": null, // stft window length in ms. + "frame_shift_ms": null, // stft window hop-lengh in ms. + "hop_length": 256, + "win_length": 1024, + "preemphasis": 0.97, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "min_level_db": -100, // normalization range + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 30,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + "signal_norm": true, // normalize the spec values in range [0, 1] + "symmetric_norm": true, // move normalization to range [-1, 1] + "clip_norm": true, // clip normalized values into the range. + "max_norm": 4, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "mel_fmin": 0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000, // maximum freq level for mel-spec. Tune for dataset!! + "do_trim_silence": false + } +} + diff --git a/vocoder/tests/test_datasets.py b/vocoder/tests/test_datasets.py new file mode 100644 index 00000000..3d6280f0 --- /dev/null +++ b/vocoder/tests/test_datasets.py @@ -0,0 +1,95 @@ +import os +import numpy as np +from torch.utils.data import DataLoader + +from TTS.vocoder.datasets.gan_dataset import GANDataset +from TTS.vocoder.datasets.preprocess import load_wav_data +from TTS.utils.audio import AudioProcessor +from TTS.utils.io import load_config + + +file_path = os.path.dirname(os.path.realpath(__file__)) +OUTPATH = os.path.join(file_path, "../../tests/outputs/loader_tests/") +os.makedirs(OUTPATH, exist_ok=True) + +C = load_config(os.path.join(file_path, 'test_config.json')) + +test_data_path = os.path.join(file_path, "../../tests/data/ljspeech/") +ok_ljspeech = os.path.exists(test_data_path) + + + +def gan_dataset_case(batch_size, seq_len, hop_len, conv_pad, return_segments, use_noise_augment, use_cache, num_workers): + ''' run dataloader with given parameters and check conditions ''' + ap = AudioProcessor(**C.audio) + eval_items, train_items = load_wav_data(test_data_path, 10) + dataset = GANDataset(ap, + train_items, + seq_len=seq_len, + hop_len=hop_len, + pad_short=2000, + conv_pad=conv_pad, + return_segments=return_segments, + use_noise_augment=use_noise_augment, + use_cache=use_cache) + loader = DataLoader(dataset=dataset, + batch_size=batch_size, + shuffle=True, + num_workers=num_workers, + pin_memory=True, + drop_last=True) + + max_iter = 10 + count_iter = 0 + + # return random segments or return the whole audio + if return_segments: + for item1, item2 in loader: + feat1, wav1 = item1 + feat2, wav2 = item2 + expected_feat_shape = (batch_size, ap.num_mels, seq_len // hop_len + conv_pad * 2) + + # check shapes + assert np.all(feat1.shape == expected_feat_shape), f" [!] {feat1.shape} vs {expected_feat_shape}" + assert (feat1.shape[2] - conv_pad * 2) * hop_len == wav1.shape[2] + + # check feature vs audio match + if not use_noise_augment: + for idx in range(batch_size): + audio = wav1[idx].squeeze() + feat = feat1[idx] + mel = ap.melspectrogram(audio) + # the first 2 and the last frame is skipped due to the padding + # applied in spec. computation. + assert (feat - mel[:, :feat1.shape[-1]])[:, 2:-1].sum() == 0, f' [!] {(feat - mel[:, :feat1.shape[-1]])[:, 2:-1].sum()}' + + count_iter += 1 + # if count_iter == max_iter: + # break + else: + for item in loader: + feat, wav = item + expected_feat_shape = (batch_size, ap.num_mels, (wav.shape[-1] // hop_len) + (conv_pad * 2)) + assert np.all(feat.shape == expected_feat_shape), f" [!] {feat.shape} vs {expected_feat_shape}" + assert (feat.shape[2] - conv_pad * 2) * hop_len == wav.shape[2] + count_iter += 1 + if count_iter == max_iter: + break + + +def test_parametrized_gan_dataset(): + ''' test dataloader with different parameters ''' + params = [ + [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], + [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 4], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, True, 0], + [1, C.audio['hop_length'], C.audio['hop_length'], 0, True, True, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, True, True, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, True, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, False, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, False, False, 0], + ] + for param in params: + print(param) + gan_dataset_case(*param) diff --git a/vocoder/tests/test_losses.py b/vocoder/tests/test_losses.py new file mode 100644 index 00000000..83314832 --- /dev/null +++ b/vocoder/tests/test_losses.py @@ -0,0 +1,62 @@ +import os +import unittest +import torch + +from TTS.vocoder.layers.losses import TorchSTFT, STFTLoss, MultiScaleSTFTLoss + +from TTS.tests import get_tests_path, get_tests_input_path, get_tests_output_path +from TTS.utils.audio import AudioProcessor +from TTS.utils.io import load_config + +TESTS_PATH = get_tests_path() + +OUT_PATH = os.path.join(get_tests_output_path(), "audio_tests") +os.makedirs(OUT_PATH, exist_ok=True) + +WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") + +file_path = os.path.dirname(os.path.realpath(__file__)) +C = load_config(os.path.join(file_path, 'test_config.json')) +ap = AudioProcessor(**C.audio) + + +def test_torch_stft(): + torch_stft = TorchSTFT(ap.n_fft, ap.hop_length, ap.win_length) + # librosa stft + wav = ap.load_wav(WAV_FILE) + M_librosa = abs(ap._stft(wav)) + # torch stft + wav = torch.from_numpy(wav[None, :]).float() + M_torch = torch_stft(wav) + # check the difference b/w librosa and torch outputs + assert (M_librosa - M_torch[0].data.numpy()).max() < 1e-5 + + +def test_stft_loss(): + stft_loss = STFTLoss(ap.n_fft, ap.hop_length, ap.win_length) + wav = ap.load_wav(WAV_FILE) + wav = torch.from_numpy(wav[None, :]).float() + loss_m, loss_sc = stft_loss(wav, wav) + assert loss_m + loss_sc == 0 + loss_m, loss_sc = stft_loss(wav, torch.rand_like(wav)) + assert loss_sc < 1.0 + assert loss_m + loss_sc > 0 + + +def test_multiscale_stft_loss(): + stft_loss = MultiScaleSTFTLoss([ap.n_fft//2, ap.n_fft, ap.n_fft*2], + [ap.hop_length // 2, ap.hop_length, ap.hop_length * 2], + [ap.win_length // 2, ap.win_length, ap.win_length * 2]) + wav = ap.load_wav(WAV_FILE) + wav = torch.from_numpy(wav[None, :]).float() + loss_m, loss_sc = stft_loss(wav, wav) + assert loss_m + loss_sc == 0 + loss_m, loss_sc = stft_loss(wav, torch.rand_like(wav)) + assert loss_sc < 1.0 + assert loss_m + loss_sc > 0 + + + + + + diff --git a/vocoder/tests/test_melgan_discriminator.py b/vocoder/tests/test_melgan_discriminator.py new file mode 100644 index 00000000..83acec8f --- /dev/null +++ b/vocoder/tests/test_melgan_discriminator.py @@ -0,0 +1,26 @@ +import numpy as np +import torch + +from TTS.vocoder.models.melgan_discriminator import MelganDiscriminator +from TTS.vocoder.models.melgan_multiscale_discriminator import MelganMultiscaleDiscriminator + + +def test_melgan_discriminator(): + model = MelganDiscriminator() + print(model) + dummy_input = torch.rand((4, 1, 256 * 10)) + output, _ = model(dummy_input) + assert np.all(output.shape == (4, 1, 10)) + + +def test_melgan_multi_scale_discriminator(): + model = MelganMultiscaleDiscriminator() + print(model) + dummy_input = torch.rand((4, 1, 256 * 16)) + scores, feats = model(dummy_input) + assert len(scores) == 3 + assert len(scores) == len(feats) + assert np.all(scores[0].shape == (4, 1, 16)) + assert np.all(feats[0][0].shape == (4, 16, 4096)) + assert np.all(feats[0][1].shape == (4, 64, 1024)) + assert np.all(feats[0][2].shape == (4, 256, 256)) diff --git a/vocoder/tests/test_melgan_generator.py b/vocoder/tests/test_melgan_generator.py new file mode 100644 index 00000000..e9c4ad60 --- /dev/null +++ b/vocoder/tests/test_melgan_generator.py @@ -0,0 +1,15 @@ +import numpy as np +import unittest +import torch + +from TTS.vocoder.models.melgan_generator import MelganGenerator + +def test_melgan_generator(): + model = MelganGenerator() + print(model) + dummy_input = torch.rand((4, 80, 64)) + output = model(dummy_input) + assert np.all(output.shape == (4, 1, 64 * 256)) + output = model.inference(dummy_input) + assert np.all(output.shape == (4, 1, (64 + 4) * 256)) + diff --git a/vocoder/tests/test_pqmf.py b/vocoder/tests/test_pqmf.py new file mode 100644 index 00000000..8444fc5a --- /dev/null +++ b/vocoder/tests/test_pqmf.py @@ -0,0 +1,33 @@ +import os +import torch +import unittest + +import numpy as np +import soundfile as sf +from librosa.core import load + +from TTS.tests import get_tests_path, get_tests_input_path, get_tests_output_path +from TTS.utils.audio import AudioProcessor +from TTS.utils.io import load_config +from TTS.vocoder.layers.pqmf import PQMF +from TTS.vocoder.layers.pqmf2 import PQMF as PQMF2 + + +TESTS_PATH = get_tests_path() +WAV_FILE = os.path.join(get_tests_input_path(), "example_1.wav") + + +def test_pqmf(): + w, sr = load(WAV_FILE) + + layer = PQMF(N=4, taps=62, cutoff=0.15, beta=9.0) + w, sr = load(WAV_FILE) + w2 = torch.from_numpy(w[None, None, :]) + b2 = layer.analysis(w2) + w2_ = layer.synthesis(b2) + + print(w2_.max()) + print(w2_.min()) + print(w2_.mean()) + sf.write('pqmf_output.wav', w2_.flatten().detach(), sr) + diff --git a/vocoder/tests/test_rwd.py b/vocoder/tests/test_rwd.py new file mode 100644 index 00000000..424d3b49 --- /dev/null +++ b/vocoder/tests/test_rwd.py @@ -0,0 +1,21 @@ +import torch +import numpy as np + +from TTS.vocoder.models.random_window_discriminator import RandomWindowDiscriminator + + +def test_rwd(): + layer = RandomWindowDiscriminator(cond_channels=80, + window_sizes=(512, 1024, 2048, 4096, + 8192), + cond_disc_downsample_factors=[ + (8, 4, 2, 2, 2), (8, 4, 2, 2), + (8, 4, 2), (8, 4), (4, 2, 2) + ], + hop_length=256) + x = torch.rand([4, 1, 22050]) + c = torch.rand([4, 80, 22050 // 256]) + + scores, _ = layer(x, c) + assert len(scores) == 10 + assert np.all(scores[0].shape == (4, 1, 1)) diff --git a/vocoder/train.py b/vocoder/train.py new file mode 100644 index 00000000..b0b4833a --- /dev/null +++ b/vocoder/train.py @@ -0,0 +1,585 @@ +import argparse +import glob +import os +import sys +import time +import traceback + +import torch +from torch.utils.data import DataLoader + +from inspect import signature + +from TTS.utils.audio import AudioProcessor +from TTS.utils.generic_utils import (KeepAverage, count_parameters, + create_experiment_folder, get_git_branch, + remove_experiment_folder, set_init_dict) +from TTS.utils.io import copy_config_file, load_config +from TTS.utils.radam import RAdam +from TTS.utils.tensorboard_logger import TensorboardLogger +from TTS.utils.training import NoamLR +from TTS.vocoder.datasets.gan_dataset import GANDataset +from TTS.vocoder.datasets.preprocess import load_wav_data +# from distribute import (DistributedSampler, apply_gradient_allreduce, +# init_distributed, reduce_tensor) +from TTS.vocoder.layers.losses import DiscriminatorLoss, GeneratorLoss +from TTS.vocoder.utils.io import save_checkpoint, save_best_model +from TTS.vocoder.utils.console_logger import ConsoleLogger +from TTS.vocoder.utils.generic_utils import (check_config, plot_results, + setup_discriminator, + setup_generator) + +torch.backends.cudnn.enabled = True +torch.backends.cudnn.benchmark = True +torch.manual_seed(54321) +use_cuda = torch.cuda.is_available() +num_gpus = torch.cuda.device_count() +print(" > Using CUDA: ", use_cuda) +print(" > Number of GPUs: ", num_gpus) + + +def setup_loader(ap, is_val=False, verbose=False): + if is_val and not c.run_eval: + loader = None + else: + dataset = GANDataset(ap=ap, + items=eval_data if is_val else train_data, + seq_len=c.seq_len, + hop_len=ap.hop_length, + pad_short=c.pad_short, + conv_pad=c.conv_pad, + is_training=not is_val, + return_segments=False if is_val else True, + use_noise_augment=c.use_noise_augment, + use_cache=c.use_cache, + verbose=verbose) + # sampler = DistributedSampler(dataset) if num_gpus > 1 else None + loader = DataLoader(dataset, + batch_size=1 if is_val else c.batch_size, + shuffle=False, + drop_last=False, + sampler=None, + num_workers=c.num_val_loader_workers + if is_val else c.num_loader_workers, + pin_memory=False) + return loader + + +def format_data(data): + if isinstance(data[0], list): + # setup input data + c_G, x_G = data[0] + c_D, x_D = data[1] + + # dispatch data to GPU + if use_cuda: + c_G = c_G.cuda(non_blocking=True) + x_G = x_G.cuda(non_blocking=True) + c_D = c_D.cuda(non_blocking=True) + x_D = x_D.cuda(non_blocking=True) + + return c_G, x_G, c_D, x_D + + # return a whole audio segment + c, x = data + if use_cuda: + c = c.cuda(non_blocking=True) + x = x.cuda(non_blocking=True) + return c, x + + +def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, + scheduler_G, scheduler_D, ap, global_step, epoch): + data_loader = setup_loader(ap, is_val=False, verbose=(epoch == 0)) + model_G.train() + model_D.train() + epoch_time = 0 + keep_avg = KeepAverage() + if use_cuda: + batch_n_iter = int( + len(data_loader.dataset) / (c.batch_size * num_gpus)) + else: + batch_n_iter = int(len(data_loader.dataset) / c.batch_size) + end_time = time.time() + c_logger.print_train_start() + for num_iter, data in enumerate(data_loader): + start_time = time.time() + + # format data + c_G, y_G, c_D, y_D = format_data(data) + loader_time = time.time() - end_time + + global_step += 1 + + # get current learning rates + current_lr_G = list(optimizer_G.param_groups)[0]['lr'] + current_lr_D = list(optimizer_D.param_groups)[0]['lr'] + + ############################## + # GENERATOR + ############################## + + # generator pass + optimizer_G.zero_grad() + y_hat = model_G(c_G) + + in_real_D = y_hat + in_fake_D = y_G + + # PQMF formatting + if y_hat.shape[1] > 1: + in_real_D = y_G + in_fake_D = model_G.pqmf_synthesis(y_hat) + y_G = model_G.pqmf_analysis(y_G) + y_hat = y_hat.view(-1, 1, y_hat.shape[2]) + y_G = y_G.view(-1, 1, y_G.shape[2]) + + if global_step > c.steps_to_start_discriminator: + + # run D with or without cond. features + if len(signature(model_D).parameters) == 2: + D_out_fake = model_D(in_fake_D, c_G) + else: + D_out_fake = model_D(in_fake_D) + D_out_real = None + + if c.use_feat_match_loss: + with torch.no_grad(): + D_out_real = model_D(in_real_D) + + # format D outputs + if isinstance(D_out_fake, tuple): + scores_fake, feats_fake = D_out_fake + if D_out_real is None: + scores_real, feats_real = None, None + else: + scores_real, feats_real = D_out_real + else: + scores_fake = D_out_fake + scores_real = D_out_real + else: + scores_fake, feats_fake, feats_real = None, None, None + + # compute losses + loss_G_dict = criterion_G(y_hat, y_G, scores_fake, feats_fake, + feats_real) + loss_G = loss_G_dict['G_loss'] + + # optimizer generator + loss_G.backward() + if c.gen_clip_grad > 0: + torch.nn.utils.clip_grad_norm_(model_G.parameters(), + c.gen_clip_grad) + optimizer_G.step() + + # setup lr + if c.noam_schedule: + scheduler_G.step() + + loss_dict = dict() + for key, value in loss_G_dict.items(): + loss_dict[key] = value.item() + + ############################## + # DISCRIMINATOR + ############################## + if global_step > c.steps_to_start_discriminator: + # discriminator pass + with torch.no_grad(): + y_hat = model_G(c_D) + + # PQMF formatting + if y_hat.shape[1] > 1: + y_hat = model_G.pqmf_synthesis(y_hat) + + optimizer_D.zero_grad() + + # run D with or without cond. features + if len(signature(model_D).parameters) == 2: + D_out_fake = model_D(y_hat.detach(), c_D) + D_out_real = model_D(y_D, c_D) + else: + D_out_fake = model_D(y_hat.detach()) + D_out_real = model_D(y_D) + + # format D outputs + if isinstance(D_out_fake, tuple): + scores_fake, feats_fake = D_out_fake + if D_out_real is None: + scores_real, feats_real = None, None + else: + scores_real, feats_real = D_out_real + else: + scores_fake = D_out_fake + scores_real = D_out_real + + # compute losses + loss_D_dict = criterion_D(scores_fake, scores_real) + loss_D = loss_D_dict['D_loss'] + + # optimizer discriminator + loss_D.backward() + if c.disc_clip_grad > 0: + torch.nn.utils.clip_grad_norm_(model_D.parameters(), + c.disc_clip_grad) + optimizer_D.step() + + # setup lr + if c.noam_schedule: + scheduler_D.step() + + for key, value in loss_D_dict.items(): + loss_dict[key] = value.item() + + step_time = time.time() - start_time + epoch_time += step_time + + # update avg stats + update_train_values = dict() + for key, value in loss_dict.items(): + update_train_values['avg_' + key] = value + update_train_values['avg_loader_time'] = loader_time + update_train_values['avg_step_time'] = step_time + keep_avg.update_values(update_train_values) + + # print training stats + if global_step % c.print_step == 0: + c_logger.print_train_step(batch_n_iter, num_iter, global_step, + step_time, loader_time, current_lr_G, + loss_dict, keep_avg.avg_values) + + # plot step stats + if global_step % 10 == 0: + iter_stats = { + "lr_G": current_lr_G, + "lr_D": current_lr_D, + "step_time": step_time + } + iter_stats.update(loss_dict) + tb_logger.tb_train_iter_stats(global_step, iter_stats) + + # save checkpoint + if global_step % c.save_step == 0: + if c.checkpoint: + # save model + save_checkpoint(model_G, + optimizer_G, + model_D, + optimizer_D, + global_step, + epoch, + OUT_PATH, + model_losses=loss_dict) + + # compute spectrograms + figures = plot_results(in_fake_D, in_real_D, ap, global_step, + 'train') + tb_logger.tb_train_figures(global_step, figures) + + # Sample audio + sample_voice = in_fake_D[0].squeeze(0).detach().cpu().numpy() + tb_logger.tb_train_audios(global_step, + {'train/audio': sample_voice}, + c.audio["sample_rate"]) + end_time = time.time() + + # print epoch stats + c_logger.print_train_epoch_end(global_step, epoch, epoch_time, keep_avg) + + # Plot Training Epoch Stats + epoch_stats = {"epoch_time": epoch_time} + epoch_stats.update(keep_avg.avg_values) + tb_logger.tb_train_epoch_stats(global_step, epoch_stats) + # TODO: plot model stats + # if c.tb_model_param_stats: + # tb_logger.tb_model_weights(model, global_step) + return keep_avg.avg_values, global_step + + +@torch.no_grad() +def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): + data_loader = setup_loader(ap, is_val=True, verbose=(epoch == 0)) + model_G.eval() + model_D.eval() + epoch_time = 0 + keep_avg = KeepAverage() + end_time = time.time() + c_logger.print_eval_start() + for num_iter, data in enumerate(data_loader): + start_time = time.time() + + # format data + c_G, y_G = format_data(data) + loader_time = time.time() - end_time + + global_step += 1 + + ############################## + # GENERATOR + ############################## + + # generator pass + y_hat = model_G(c_G) + + in_real_D = y_hat + in_fake_D = y_G + + # PQMF formatting + if y_hat.shape[1] > 1: + in_real_D = y_G + in_fake_D = model_G.pqmf_synthesis(y_hat) + y_G = model_G.pqmf_analysis(y_G) + y_hat = y_hat.view(-1, 1, y_hat.shape[2]) + y_G = y_G.view(-1, 1, y_G.shape[2]) + + D_out_fake = model_D(in_fake_D) + D_out_real = None + if c.use_feat_match_loss: + with torch.no_grad(): + D_out_real = model_D(in_real_D) + + # format D outputs + if isinstance(D_out_fake, tuple): + scores_fake, feats_fake = D_out_fake + if D_out_real is None: + feats_real = None + else: + _, feats_real = D_out_real + else: + scores_fake = D_out_fake + + # compute losses + loss_G_dict = criterion_G(y_hat, y_G, scores_fake, feats_fake, + feats_real) + + loss_dict = dict() + for key, value in loss_G_dict.items(): + loss_dict[key] = value.item() + + step_time = time.time() - start_time + epoch_time += step_time + + # update avg stats + update_eval_values = dict() + for key, value in loss_G_dict.items(): + update_eval_values['avg_' + key] = value.item() + update_eval_values['avg_loader_time'] = loader_time + update_eval_values['avg_step_time'] = step_time + keep_avg.update_values(update_eval_values) + + # print eval stats + if c.print_eval: + c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) + + # compute spectrograms + figures = plot_results(y_hat, y_G, ap, global_step, 'eval') + tb_logger.tb_eval_figures(global_step, figures) + + # Sample audio + sample_voice = y_hat[0].squeeze(0).detach().cpu().numpy() + tb_logger.tb_eval_audios(global_step, {'eval/audio': sample_voice}, + c.audio["sample_rate"]) + + # synthesize a full voice + data_loader.return_segments = False + + return keep_avg.avg_values + + +# FIXME: move args definition/parsing inside of main? +def main(args): # pylint: disable=redefined-outer-name + # pylint: disable=global-variable-undefined + global train_data, eval_data + eval_data, train_data = load_wav_data(c.data_path, c.eval_split_size) + + # setup audio processor + ap = AudioProcessor(**c.audio) + # DISTRUBUTED + # if num_gpus > 1: + # init_distributed(args.rank, num_gpus, args.group_id, + # c.distributed["backend"], c.distributed["url"]) + + # setup models + model_gen = setup_generator(c) + model_disc = setup_discriminator(c) + + # setup optimizers + optimizer_gen = RAdam(model_gen.parameters(), lr=c.lr_gen, weight_decay=0) + optimizer_disc = RAdam(model_disc.parameters(), + lr=c.lr_disc, + weight_decay=0) + + # setup criterion + criterion_gen = GeneratorLoss(c) + criterion_disc = DiscriminatorLoss(c) + + if args.restore_path: + checkpoint = torch.load(args.restore_path, map_location='cpu') + try: + model_gen.load_state_dict(checkpoint['model']) + optimizer_gen.load_state_dict(checkpoint['optimizer']) + model_disc.load_state_dict(checkpoint['model_disc']) + optimizer_disc.load_state_dict(checkpoint['optimizer_disc']) + except: + print(" > Partial model initialization.") + model_dict = model_gen.state_dict() + model_dict = set_init_dict(model_dict, checkpoint['model'], c) + model_gen.load_state_dict(model_dict) + + model_dict = model_disc.state_dict() + model_dict = set_init_dict(model_dict, checkpoint['model_disc'], c) + model_disc.load_state_dict(model_dict) + del model_dict + + # reset lr if not countinuining training. + for group in optimizer_gen.param_groups: + group['lr'] = c.lr_gen + + for group in optimizer_disc.param_groups: + group['lr'] = c.lr_disc + + print(" > Model restored from step %d" % checkpoint['step'], + flush=True) + args.restore_step = checkpoint['step'] + else: + args.restore_step = 0 + + if use_cuda: + model_gen.cuda() + criterion_gen.cuda() + model_disc.cuda() + criterion_disc.cuda() + + # DISTRUBUTED + # if num_gpus > 1: + # model = apply_gradient_allreduce(model) + + if c.noam_schedule: + scheduler_gen = NoamLR(optimizer_gen, + warmup_steps=c.warmup_steps_gen, + last_epoch=args.restore_step - 1) + scheduler_disc = NoamLR(optimizer_disc, + warmup_steps=c.warmup_steps_gen, + last_epoch=args.restore_step - 1) + else: + scheduler_gen, scheduler_disc = None, None + + num_params = count_parameters(model_gen) + print(" > Generator has {} parameters".format(num_params), flush=True) + num_params = count_parameters(model_disc) + print(" > Discriminator has {} parameters".format(num_params), flush=True) + + if 'best_loss' not in locals(): + best_loss = float('inf') + + global_step = args.restore_step + for epoch in range(0, c.epochs): + c_logger.print_epoch_start(epoch, c.epochs) + _, global_step = train(model_gen, criterion_gen, optimizer_gen, + model_disc, criterion_disc, optimizer_disc, + scheduler_gen, scheduler_disc, ap, global_step, + epoch) + eval_avg_loss_dict = evaluate(model_gen, criterion_gen, model_disc, ap, + global_step, epoch) + c_logger.print_epoch_end(epoch, eval_avg_loss_dict) + target_loss = eval_avg_loss_dict[c.target_loss] + best_loss = save_best_model(target_loss, + best_loss, + model_gen, + optimizer_gen, + model_disc, + optimizer_disc, + global_step, + epoch, + OUT_PATH, + model_losses=eval_avg_loss_dict) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--continue_path', + type=str, + help= + 'Training output folder to continue training. Use to continue a training. If it is used, "config_path" is ignored.', + default='', + required='--config_path' not in sys.argv) + parser.add_argument( + '--restore_path', + type=str, + help='Model file to be restored. Use to finetune a model.', + default='') + parser.add_argument('--config_path', + type=str, + help='Path to config file for training.', + required='--continue_path' not in sys.argv) + parser.add_argument('--debug', + type=bool, + default=False, + help='Do not verify commit integrity to run training.') + + # DISTRUBUTED + parser.add_argument( + '--rank', + type=int, + default=0, + help='DISTRIBUTED: process rank for distributed training.') + parser.add_argument('--group_id', + type=str, + default="", + help='DISTRIBUTED: process group id.') + args = parser.parse_args() + + if args.continue_path != '': + args.output_path = args.continue_path + args.config_path = os.path.join(args.continue_path, 'config.json') + list_of_files = glob.glob( + args.continue_path + + "/*.pth.tar") # * means all if need specific format then *.csv + latest_model_file = max(list_of_files, key=os.path.getctime) + args.restore_path = latest_model_file + print(f" > Training continues for {args.restore_path}") + + # setup output paths and read configs + c = load_config(args.config_path) + check_config(c) + _ = os.path.dirname(os.path.realpath(__file__)) + + OUT_PATH = args.continue_path + if args.continue_path == '': + OUT_PATH = create_experiment_folder(c.output_path, c.run_name, + args.debug) + + AUDIO_PATH = os.path.join(OUT_PATH, 'test_audios') + + c_logger = ConsoleLogger() + + if args.rank == 0: + os.makedirs(AUDIO_PATH, exist_ok=True) + new_fields = {} + if args.restore_path: + new_fields["restore_path"] = args.restore_path + new_fields["github_branch"] = get_git_branch() + copy_config_file(args.config_path, + os.path.join(OUT_PATH, 'config.json'), new_fields) + os.chmod(AUDIO_PATH, 0o775) + os.chmod(OUT_PATH, 0o775) + + LOG_DIR = OUT_PATH + tb_logger = TensorboardLogger(LOG_DIR, model_name='VOCODER') + + # write model desc to tensorboard + tb_logger.tb_add_text('model-description', c['run_description'], 0) + + try: + main(args) + except KeyboardInterrupt: + remove_experiment_folder(OUT_PATH) + try: + sys.exit(0) + except SystemExit: + os._exit(0) # pylint: disable=protected-access + except Exception: # pylint: disable=broad-except + remove_experiment_folder(OUT_PATH) + traceback.print_exc() + sys.exit(1) diff --git a/vocoder/utils/__init__.py b/vocoder/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/vocoder/utils/console_logger.py b/vocoder/utils/console_logger.py new file mode 100644 index 00000000..4fe132bb --- /dev/null +++ b/vocoder/utils/console_logger.py @@ -0,0 +1,97 @@ +import datetime +from TTS.utils.io import AttrDict + + +tcolors = AttrDict({ + 'OKBLUE': '\033[94m', + 'HEADER': '\033[95m', + 'OKGREEN': '\033[92m', + 'WARNING': '\033[93m', + 'FAIL': '\033[91m', + 'ENDC': '\033[0m', + 'BOLD': '\033[1m', + 'UNDERLINE': '\033[4m' +}) + + +class ConsoleLogger(): + def __init__(self): + # TODO: color code for value changes + # use these to compare values between iterations + self.old_train_loss_dict = None + self.old_epoch_loss_dict = None + self.old_eval_loss_dict = None + + # pylint: disable=no-self-use + def get_time(self): + now = datetime.datetime.now() + return now.strftime("%Y-%m-%d %H:%M:%S") + + def print_epoch_start(self, epoch, max_epoch): + print("\n{}{} > EPOCH: {}/{}{}".format(tcolors.UNDERLINE, tcolors.BOLD, + epoch, max_epoch, tcolors.ENDC), + flush=True) + + def print_train_start(self): + print(f"\n{tcolors.BOLD} > TRAINING ({self.get_time()}) {tcolors.ENDC}") + + def print_train_step(self, batch_steps, step, global_step, + step_time, loader_time, lr, + loss_dict, avg_loss_dict): + indent = " | > " + print() + log_text = "{} --> STEP: {}/{} -- GLOBAL_STEP: {}{}\n".format( + tcolors.BOLD, step, batch_steps, global_step, tcolors.ENDC) + for key, value in loss_dict.items(): + # print the avg value if given + if f'avg_{key}' in avg_loss_dict.keys(): + log_text += "{}{}: {:.5f} ({:.5f})\n".format(indent, key, value, avg_loss_dict[f'avg_{key}']) + else: + log_text += "{}{}: {:.5f} \n".format(indent, key, value) + log_text += f"{indent}step_time: {step_time:.2f}\n{indent}loader_time: {loader_time:.2f}\n{indent}lr: {lr:.5f}" + print(log_text, flush=True) + + # pylint: disable=unused-argument + def print_train_epoch_end(self, global_step, epoch, epoch_time, + print_dict): + indent = " | > " + log_text = f"\n{tcolors.BOLD} --> TRAIN PERFORMACE -- EPOCH TIME: {epoch} sec -- GLOBAL_STEP: {global_step}{tcolors.ENDC}\n" + for key, value in print_dict.items(): + log_text += "{}{}: {:.5f}\n".format(indent, key, value) + print(log_text, flush=True) + + def print_eval_start(self): + print(f"{tcolors.BOLD} > EVALUATION {tcolors.ENDC}\n") + + def print_eval_step(self, step, loss_dict, avg_loss_dict): + indent = " | > " + print() + log_text = f"{tcolors.BOLD} --> STEP: {step}{tcolors.ENDC}\n" + for key, value in loss_dict.items(): + # print the avg value if given + if f'avg_{key}' in avg_loss_dict.keys(): + log_text += "{}{}: {:.5f} ({:.5f})\n".format(indent, key, value, avg_loss_dict[f'avg_{key}']) + else: + log_text += "{}{}: {:.5f} \n".format(indent, key, value) + print(log_text, flush=True) + + def print_epoch_end(self, epoch, avg_loss_dict): + indent = " | > " + log_text = " {}--> EVAL PERFORMANCE{}\n".format( + tcolors.BOLD, tcolors.ENDC) + for key, value in avg_loss_dict.items(): + # print the avg value if given + color = '' + sign = '+' + diff = 0 + if self.old_eval_loss_dict is not None: + diff = value - self.old_eval_loss_dict[key] + if diff < 0: + color = tcolors.OKGREEN + sign = '' + elif diff > 0: + color = tcolors.FAIL + sing = '+' + log_text += "{}{}:{} {:.5f} {}({}{:.5f})\n".format(indent, key, color, value, tcolors.ENDC, sign, diff) + self.old_eval_loss_dict = avg_loss_dict + print(log_text, flush=True) diff --git a/vocoder/utils/generic_utils.py b/vocoder/utils/generic_utils.py new file mode 100644 index 00000000..062de160 --- /dev/null +++ b/vocoder/utils/generic_utils.py @@ -0,0 +1,102 @@ +import re +import importlib +import numpy as np +from matplotlib import pyplot as plt + +from TTS.utils.visual import plot_spectrogram + + +def plot_results(y_hat, y, ap, global_step, name_prefix): + """ Plot vocoder model results """ + + # select an instance from batch + y_hat = y_hat[0].squeeze(0).detach().cpu().numpy() + y = y[0].squeeze(0).detach().cpu().numpy() + + spec_fake = ap.spectrogram(y_hat).T + spec_real = ap.spectrogram(y).T + spec_diff = np.abs(spec_fake - spec_real) + + # plot figure and save it + fig_wave = plt.figure() + plt.subplot(2, 1, 1) + plt.plot(y) + plt.title("groundtruth speech") + plt.subplot(2, 1, 2) + plt.plot(y_hat) + plt.title(f"generated speech @ {global_step} steps") + plt.tight_layout() + plt.close() + + figures = { + name_prefix + "/spectrogram/fake": plot_spectrogram(spec_fake, ap), + name_prefix + "spectrogram/real": plot_spectrogram(spec_real, ap), + name_prefix + "spectrogram/diff": plot_spectrogram(spec_diff, ap), + name_prefix + "speech_comparison": fig_wave, + } + return figures + + +def to_camel(text): + text = text.capitalize() + return re.sub(r'(?!^)_([a-zA-Z])', lambda m: m.group(1).upper(), text) + + +def setup_generator(c): + print(" > Generator Model: {}".format(c.generator_model)) + MyModel = importlib.import_module('TTS.vocoder.models.' + + c.generator_model.lower()) + MyModel = getattr(MyModel, to_camel(c.generator_model)) + if c.generator_model in 'melgan_generator': + model = MyModel( + in_channels=c.audio['num_mels'], + out_channels=1, + proj_kernel=7, + base_channels=512, + upsample_factors=c.generator_model_params['upsample_factors'], + res_kernel=3, + num_res_blocks=c.generator_model_params['num_res_blocks']) + if c.generator_model in 'melgan_fb_generator': + pass + if c.generator_model in 'multiband_melgan_generator': + model = MyModel( + in_channels=c.audio['num_mels'], + out_channels=4, + proj_kernel=7, + base_channels=384, + upsample_factors=c.generator_model_params['upsample_factors'], + res_kernel=3, + num_res_blocks=c.generator_model_params['num_res_blocks']) + return model + + +def setup_discriminator(c): + print(" > Discriminator Model: {}".format(c.discriminator_model)) + MyModel = importlib.import_module('TTS.vocoder.models.' + + c.discriminator_model.lower()) + MyModel = getattr(MyModel, to_camel(c.discriminator_model)) + if c.discriminator_model in 'random_window_discriminator': + model = MyModel( + cond_channels=c.audio['num_mels'], + hop_length=c.audio['hop_length'], + uncond_disc_donwsample_factors=c. + discriminator_model_params['uncond_disc_donwsample_factors'], + cond_disc_downsample_factors=c. + discriminator_model_params['cond_disc_downsample_factors'], + cond_disc_out_channels=c. + discriminator_model_params['cond_disc_out_channels'], + window_sizes=c.discriminator_model_params['window_sizes']) + if c.discriminator_model in 'melgan_multiscale_discriminator': + model = MyModel( + in_channels=1, + out_channels=1, + kernel_sizes=(5, 3), + base_channels=c.discriminator_model_params['base_channels'], + max_channels=c.discriminator_model_params['max_channels'], + downsample_factors=c. + discriminator_model_params['downsample_factors']) + return model + + +# def check_config(c): +# pass \ No newline at end of file diff --git a/vocoder/utils/io.py b/vocoder/utils/io.py new file mode 100644 index 00000000..9526b9db --- /dev/null +++ b/vocoder/utils/io.py @@ -0,0 +1,52 @@ +import os +import torch +import datetime + + +def save_model(model, optimizer, model_disc, optimizer_disc, current_step, + epoch, output_path, **kwargs): + model_state = model.state_dict() + model_disc_state = model_disc.state_dict() + optimizer_state = optimizer.state_dict() if optimizer is not None else None + optimizer_disc_state = optimizer_disc.state_dict( + ) if optimizer_disc is not None else None + state = { + 'model': model_state, + 'optimizer': optimizer_state, + 'model_disc': model_disc_state, + 'optimizer_disc': optimizer_disc_state, + 'step': current_step, + 'epoch': epoch, + 'date': datetime.date.today().strftime("%B %d, %Y"), + } + state.update(kwargs) + torch.save(state, output_path) + + +def save_checkpoint(model, optimizer, model_disc, optimizer_disc, current_step, + epoch, output_folder, **kwargs): + file_name = 'checkpoint_{}.pth.tar'.format(current_step) + checkpoint_path = os.path.join(output_folder, file_name) + print(" > CHECKPOINT : {}".format(checkpoint_path)) + save_model(model, optimizer, model_disc, optimizer_disc, current_step, + epoch, checkpoint_path, **kwargs) + + +def save_best_model(target_loss, best_loss, model, optimizer, model_disc, + optimizer_disc, current_step, epoch, output_folder, + **kwargs): + if target_loss < best_loss: + file_name = 'best_model.pth.tar' + checkpoint_path = os.path.join(output_folder, file_name) + print(" > BEST MODEL : {}".format(checkpoint_path)) + save_model(model, + optimizer, + model_disc, + optimizer_disc, + current_step, + epoch, + checkpoint_path, + model_loss=target_loss, + **kwargs) + best_loss = target_loss + return best_loss \ No newline at end of file From 8e7a15f07b20d0b3134d8c33cd4e582de30db10e Mon Sep 17 00:00:00 2001 From: erogol Date: Sun, 31 May 2020 03:31:13 +0200 Subject: [PATCH 06/68] version update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 84a31488..e8a626c6 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ args, unknown_args = parser.parse_known_args() # Remove our arguments from argv so that setuptools doesn't see them sys.argv = [sys.argv[0]] + unknown_args -version = '0.0.2' +version = '0.1.0' # Adapted from https://github.com/pytorch/pytorch cwd = os.path.dirname(os.path.abspath(__file__)) From d5bbcd0487be1b87d5d75b78608a44e19f868b68 Mon Sep 17 00:00:00 2001 From: erogol Date: Sun, 31 May 2020 03:31:42 +0200 Subject: [PATCH 07/68] plot fix --- vocoder/configs/melgan_config.json | 7 +++---- vocoder/train.py | 4 ++-- vocoder/utils/generic_utils.py | 8 ++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json index 9a3ded37..7da42f75 100644 --- a/vocoder/configs/melgan_config.json +++ b/vocoder/configs/melgan_config.json @@ -109,14 +109,13 @@ "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. // OPTIMIZER - "noam_schedule": true, // use noam warmup and lr schedule. - "grad_clip": 1.0, // upper limit for gradients for clipping. + "noam_schedule": false, // use noam warmup and lr schedule. + "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "warmup_steps_disc": 4000, "epochs": 1000, // total number of epochs to train. "wd": 0.000001, // Weight decay weight. "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. "lr_disc": 0.0001, - "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "warmup_steps_disc": 4000, "gen_clip_grad": 10.0, "disc_clip_grad": 10.0, diff --git a/vocoder/train.py b/vocoder/train.py index b0b4833a..0d039e73 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -372,11 +372,11 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) # compute spectrograms - figures = plot_results(y_hat, y_G, ap, global_step, 'eval') + figures = plot_results(in_fake_D, in_real_D, ap, global_step, 'eval') tb_logger.tb_eval_figures(global_step, figures) # Sample audio - sample_voice = y_hat[0].squeeze(0).detach().cpu().numpy() + sample_voice = in_fake_D[0].squeeze(0).detach().cpu().numpy() tb_logger.tb_eval_audios(global_step, {'eval/audio': sample_voice}, c.audio["sample_rate"]) diff --git a/vocoder/utils/generic_utils.py b/vocoder/utils/generic_utils.py index 062de160..179c4064 100644 --- a/vocoder/utils/generic_utils.py +++ b/vocoder/utils/generic_utils.py @@ -13,8 +13,8 @@ def plot_results(y_hat, y, ap, global_step, name_prefix): y_hat = y_hat[0].squeeze(0).detach().cpu().numpy() y = y[0].squeeze(0).detach().cpu().numpy() - spec_fake = ap.spectrogram(y_hat).T - spec_real = ap.spectrogram(y).T + spec_fake = ap.melspectrogram(y_hat).T + spec_real = ap.melspectrogram(y).T spec_diff = np.abs(spec_fake - spec_real) # plot figure and save it @@ -98,5 +98,5 @@ def setup_discriminator(c): return model -# def check_config(c): -# pass \ No newline at end of file +def check_config(c): + pass \ No newline at end of file From 3f789187159e5bd385f49f26b8643e801d5838a5 Mon Sep 17 00:00:00 2001 From: ForceCore Date: Sun, 31 May 2020 19:51:16 +0900 Subject: [PATCH 08/68] Fix phoneme cache file name aliasing problem When the wav file has multiple dots in the file name, _load_or_generate_phoneme_sequence would only use only the first segment of the file name and cause overwrite of *_phoneme.npy --- datasets/TTSDataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/TTSDataset.py b/datasets/TTSDataset.py index 0d884c00..7fe966d7 100644 --- a/datasets/TTSDataset.py +++ b/datasets/TTSDataset.py @@ -92,7 +92,7 @@ class MyDataset(Dataset): return phonemes def _load_or_generate_phoneme_sequence(self, wav_file, text): - file_name = os.path.basename(wav_file).split('.')[0] + file_name = os.path.splitext(os.path.basename(wav_file))[0] cache_path = os.path.join(self.phoneme_cache_path, file_name + '_phoneme.npy') try: From 29915ba85c77db944071ea1e0be2b153c2e74102 Mon Sep 17 00:00:00 2001 From: erogol Date: Sun, 31 May 2020 14:18:29 +0200 Subject: [PATCH 09/68] bug fix --- vocoder/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vocoder/train.py b/vocoder/train.py index 0d039e73..a30e3bff 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -137,7 +137,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, if global_step > c.steps_to_start_discriminator: # run D with or without cond. features - if len(signature(model_D).parameters) == 2: + if len(signature(model_D.forward).parameters) == 2: D_out_fake = model_D(in_fake_D, c_G) else: D_out_fake = model_D(in_fake_D) @@ -195,7 +195,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, optimizer_D.zero_grad() # run D with or without cond. features - if len(signature(model_D).parameters) == 2: + if len(signature(model_D.forward).parameters) == 2: D_out_fake = model_D(y_hat.detach(), c_D) D_out_real = model_D(y_D, c_D) else: From bb6729b42ad3b46eeaf95ca5243159ef62e1f899 Mon Sep 17 00:00:00 2001 From: erogol Date: Sun, 31 May 2020 18:56:57 +0200 Subject: [PATCH 10/68] add multiband melgan config and rename loss weights for vocoder --- vocoder/configs/melgan_config.json | 8 ++++---- vocoder/layers/losses.py | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json index 7da42f75..b97206fa 100644 --- a/vocoder/configs/melgan_config.json +++ b/vocoder/configs/melgan_config.json @@ -53,10 +53,10 @@ "use_hinge_gan_loss": false, "use_feat_match_loss": false, // use only with melgan discriminators - "stft_loss_alpha": 1, - "mse_gan_loss_alpha": 1, - "hinge_gan_loss_alpha": 1, - "feat_match_loss_alpha": 10.0, + "stft_loss_weight": 1, + "mse_gan_loss_weight": 2.5, + "hinge_gan_loss_weight": 2.5, + "feat_match_loss_weight": 10.0, "stft_loss_params": { "n_ffts": [1024, 2048, 512], diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index 11985629..27091c0a 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -149,10 +149,10 @@ class GeneratorLoss(nn.Module): self.use_hinge_gan_loss = C.use_hinge_gan_loss self.use_feat_match_loss = C.use_feat_match_loss - self.stft_loss_alpha = C.stft_loss_alpha - self.mse_gan_loss_alpha = C.mse_gan_loss_alpha - self.hinge_gan_loss_alpha = C.hinge_gan_loss_alpha - self.feat_match_loss_alpha = C.feat_match_loss_alpha + self.stft_loss_weight = C.stft_loss_weight + self.mse_gan_loss_weight = C.mse_gan_loss_weight + self.hinge_gan_loss_weight = C.hinge_gan_loss_weight + self.feat_match_loss_weight = C.feat_match_loss_weight if C.use_stft_loss: self.stft_loss = MultiScaleSTFTLoss(**C.stft_loss_params) @@ -172,7 +172,7 @@ class GeneratorLoss(nn.Module): stft_loss_mg, stft_loss_sc = self.stft_loss(y_hat.squeeze(1), y.squeeze(1)) return_dict['G_stft_loss_mg'] = stft_loss_mg return_dict['G_stft_loss_sc'] = stft_loss_sc - loss += self.stft_loss_alpha * (stft_loss_mg + stft_loss_sc) + loss += self.stft_loss_weight * (stft_loss_mg + stft_loss_sc) # Fake Losses if self.use_mse_gan_loss and scores_fake is not None: @@ -185,7 +185,7 @@ class GeneratorLoss(nn.Module): fake_loss = self.mse_loss(scores_fake) mse_fake_loss = fake_loss return_dict['G_mse_fake_loss'] = mse_fake_loss - loss += self.mse_gan_loss_alpha * mse_fake_loss + loss += self.mse_gan_loss_weight * mse_fake_loss if self.use_hinge_gan_loss and not scores_fake is not None: hinge_fake_loss = 0 @@ -197,13 +197,13 @@ class GeneratorLoss(nn.Module): fake_loss = self.hinge_loss(scores_fake) hinge_fake_loss = fake_loss return_dict['G_hinge_fake_loss'] = hinge_fake_loss - loss += self.hinge_gan_loss_alpha * hinge_fake_loss + loss += self.hinge_gan_loss_weight * hinge_fake_loss # Feature Matching Loss if self.use_feat_match_loss and not feats_fake: feat_match_loss = self.feat_match_loss(feats_fake, feats_real) return_dict['G_feat_match_loss'] = feat_match_loss - loss += self.feat_match_loss_alpha * feat_match_loss + loss += self.feat_match_loss_weight * feat_match_loss return_dict['G_loss'] = loss return return_dict @@ -217,8 +217,8 @@ class DiscriminatorLoss(nn.Module): self.use_mse_gan_loss = C.use_mse_gan_loss self.use_hinge_gan_loss = C.use_hinge_gan_loss - self.mse_gan_loss_alpha = C.mse_gan_loss_alpha - self.hinge_gan_loss_alpha = C.hinge_gan_loss_alpha + self.mse_gan_loss_weight = C.mse_gan_loss_weight + self.hinge_gan_loss_weight = C.hinge_gan_loss_weight if C.use_mse_gan_loss: self.mse_loss = MSEDLoss() @@ -247,7 +247,7 @@ class DiscriminatorLoss(nn.Module): return_dict['D_mse_gan_loss'] = mse_gan_loss return_dict['D_mse_gan_real_loss'] = mse_gan_real_loss return_dict['D_mse_gan_fake_loss'] = mse_gan_fake_loss - loss += self.mse_gan_loss_alpha * mse_gan_loss + loss += self.mse_gan_loss_weight * mse_gan_loss if self.use_hinge_gan_loss: hinge_gan_loss = 0 @@ -267,7 +267,7 @@ class DiscriminatorLoss(nn.Module): return_dict['D_hinge_gan_loss'] = hinge_gan_loss return_dict['D_hinge_gan_real_loss'] = hinge_gan_real_loss return_dict['D_hinge_gan_fake_loss'] = hinge_gan_fake_loss - loss += self.hinge_gan_loss_alpha * hinge_gan_loss + loss += self.hinge_gan_loss_weight * hinge_gan_loss return_dict['D_loss'] = loss return return_dict \ No newline at end of file From 1a470ba6b94914a4170e4687639d58c4ba9efb81 Mon Sep 17 00:00:00 2001 From: erogol Date: Sun, 31 May 2020 19:03:07 +0200 Subject: [PATCH 11/68] multiband rwd D config --- .../multiband-melgan_and_rwd_config.json | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 vocoder/configs/multiband-melgan_and_rwd_config.json diff --git a/vocoder/configs/multiband-melgan_and_rwd_config.json b/vocoder/configs/multiband-melgan_and_rwd_config.json new file mode 100644 index 00000000..9aeb7933 --- /dev/null +++ b/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -0,0 +1,136 @@ +{ + "run_name": "multiband-melgan-rwd", + "run_description": "multibadn melgan with random window discriminator", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // DISTRIBUTED TRAINING + // "distributed":{ + // "backend": "nccl", + // "url": "tcp:\/\/localhost:54321" + // }, + + // MODEL PARAMETERS + "use_pqmf": true, + + // LOSS PARAMETERS + "use_stft_loss": true, + "use_mse_gan_loss": true, + "use_hinge_gan_loss": false, + "use_feat_match_loss": false, // use only with melgan discriminators + + "stft_loss_weight": 1, + "mse_gan_loss_weight": 2.5, + "hinge_gan_loss_weight": 1, + "feat_match_loss_weight": 10.0, + + "stft_loss_params": { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }, + "target_loss": "avg_G_loss", // loss value to pick the best model + + // DISCRIMINATOR + // "discriminator_model": "melgan_multiscale_discriminator", + // "discriminator_model_params":{ + // "base_channels": 16, + // "max_channels":1024, + // "downsample_factors":[4, 4, 4, 4] + // }, + "steps_to_start_discriminator": 100000, // steps required to start GAN trainining.1 + + "discriminator_model": "random_window_discriminator", + "discriminator_model_params":{ + "uncond_disc_donwsample_factors": [8, 4], + "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]], + "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], + "window_sizes": [512, 1024, 2048, 4096, 8192] + }, + + // GENERATOR + "generator_model": "multiband_melgan_generator", + "generator_model_params": { + "upsample_factors":[2 ,2, 4, 4], + "num_res_blocks": 4 + }, + + // DATASET + "data_path": "/root/LJSpeech-1.1/wavs/", + "seq_len": 16384, + "pad_short": 2000, + "conv_pad": 0, + "use_noise_augment": true, + "use_cache": true, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": false, // use noam warmup and lr schedule. + "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "warmup_steps_disc": 4000, + "epochs": 1000, // total number of epochs to train. + "wd": 0.000001, // Weight decay weight. + "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_disc": 0.0001, + "gen_clip_grad": 10.0, + "disc_clip_grad": 10.0, + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log traning on console. + "print_eval": false, // If True, it prints intermediate loss values in evalulation. + "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "eval_split_size": 10, + + // PATHS + "output_path": "/home/erogol/Models/LJSpeech/" +} + From 5662aa3bbe4d73120a89b4eb59a3ce8a03698abd Mon Sep 17 00:00:00 2001 From: erogol Date: Mon, 1 Jun 2020 03:02:51 +0200 Subject: [PATCH 12/68] tb eval print --- vocoder/train.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vocoder/train.py b/vocoder/train.py index a30e3bff..27e14e8a 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -383,6 +383,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): # synthesize a full voice data_loader.return_segments = False + tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) return keep_avg.avg_values From 4619b84ddbda458b202e74f28db9a02191c0a499 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 17:07:15 +0200 Subject: [PATCH 13/68] multiscale subband stft loss and missing tb_logger for eval stats --- vocoder/configs/melgan_config.json | 8 +++++ vocoder/layers/losses.py | 21 ++++++++++-- vocoder/train.py | 53 ++++++++++++++---------------- 3 files changed, 51 insertions(+), 31 deletions(-) diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json index b97206fa..4aeba82e 100644 --- a/vocoder/configs/melgan_config.json +++ b/vocoder/configs/melgan_config.json @@ -49,11 +49,13 @@ // LOSS PARAMETERS "use_stft_loss": true, + "use_subband_stft_loss": true, "use_mse_gan_loss": true, "use_hinge_gan_loss": false, "use_feat_match_loss": false, // use only with melgan discriminators "stft_loss_weight": 1, + "subband_stft_loss_weight": 1, "mse_gan_loss_weight": 2.5, "hinge_gan_loss_weight": 2.5, "feat_match_loss_weight": 10.0, @@ -63,6 +65,12 @@ "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240] }, + "subband_stft_loss_params":{ + "n_ffts": [384, 683, 171], + "hop_lengths": [30, 60, 10], + "win_lengths": [150, 300, 60] + }, + "target_loss": "avg_G_loss", // loss value to pick the best model // DISCRIMINATOR diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index 27091c0a..e242d209 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -49,7 +49,6 @@ class STFTLoss(nn.Module): loss_sc = torch.norm(y_M - y_hat_M, p="fro") / torch.norm(y_M, p="fro") return loss_mag, loss_sc - class MultiScaleSTFTLoss(torch.nn.Module): def __init__(self, n_ffts=[1024, 2048, 512], @@ -73,6 +72,13 @@ class MultiScaleSTFTLoss(torch.nn.Module): return loss_mag, loss_sc +class MultiScaleSubbandSTFTLoss(MultiScaleSTFTLoss): + def forward(self, y_hat, y): + y_hat = y_hat.view(-1, 1, y_hat.shape[2]) + y = y.view(-1, 1, y.shape[2]) + return super().forward(y_hat.squeeze(1), y.squeeze(1)) + + class MSEGLoss(nn.Module): """ Mean Squared Generator Loss """ def __init__(self,): @@ -145,17 +151,21 @@ class GeneratorLoss(nn.Module): " [!] Cannot use HingeGANLoss and MSEGANLoss together." self.use_stft_loss = C.use_stft_loss + self.use_subband_stft_loss = C.use_subband_stft_loss self.use_mse_gan_loss = C.use_mse_gan_loss self.use_hinge_gan_loss = C.use_hinge_gan_loss self.use_feat_match_loss = C.use_feat_match_loss self.stft_loss_weight = C.stft_loss_weight + self.subband_stft_loss_weight = C.subband_stft_loss_weight self.mse_gan_loss_weight = C.mse_gan_loss_weight self.hinge_gan_loss_weight = C.hinge_gan_loss_weight self.feat_match_loss_weight = C.feat_match_loss_weight if C.use_stft_loss: self.stft_loss = MultiScaleSTFTLoss(**C.stft_loss_params) + if C.use_subband_stft_loss: + self.subband_stft_loss = MultiScaleSubbandSTFTLoss(**C.subband_stft_loss_params) if C.use_mse_gan_loss: self.mse_loss = MSEGLoss() if C.use_hinge_gan_loss: @@ -163,7 +173,7 @@ class GeneratorLoss(nn.Module): if C.use_feat_match_loss: self.feat_match_loss = MelganFeatureLoss() - def forward(self, y_hat=None, y=None, scores_fake=None, feats_fake=None, feats_real=None): + def forward(self, y_hat=None, y=None, scores_fake=None, feats_fake=None, feats_real=None, y_hat_sub=None, y_sub=None): loss = 0 return_dict = {} @@ -174,6 +184,13 @@ class GeneratorLoss(nn.Module): return_dict['G_stft_loss_sc'] = stft_loss_sc loss += self.stft_loss_weight * (stft_loss_mg + stft_loss_sc) + # subband STFT Loss + if self.use_subband_stft_loss: + subband_stft_loss_mg, subband_stft_loss_sc = self.subband_stft_loss(y_hat_sub, y_sub) + return_dict['G_subband_stft_loss_mg'] = subband_stft_loss_mg + return_dict['G_subband_stft_loss_sc'] = subband_stft_loss_sc + loss += self.subband_stft_loss_weight * (subband_stft_loss_mg + subband_stft_loss_sc) + # Fake Losses if self.use_mse_gan_loss and scores_fake is not None: mse_fake_loss = 0 diff --git a/vocoder/train.py b/vocoder/train.py index 27e14e8a..6341bb6c 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -122,30 +122,27 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, # generator pass optimizer_G.zero_grad() y_hat = model_G(c_G) - - in_real_D = y_hat - in_fake_D = y_G + y_hat_sub = None + y_G_sub = None # PQMF formatting if y_hat.shape[1] > 1: - in_real_D = y_G - in_fake_D = model_G.pqmf_synthesis(y_hat) - y_G = model_G.pqmf_analysis(y_G) - y_hat = y_hat.view(-1, 1, y_hat.shape[2]) - y_G = y_G.view(-1, 1, y_G.shape[2]) + y_hat_sub = y_hat + y_hat = model_G.pqmf_synthesis(y_hat) + y_G_sub = model_G.pqmf_analysis(y_G) if global_step > c.steps_to_start_discriminator: # run D with or without cond. features if len(signature(model_D.forward).parameters) == 2: - D_out_fake = model_D(in_fake_D, c_G) + D_out_fake = model_D(y_hat, c_G) else: - D_out_fake = model_D(in_fake_D) + D_out_fake = model_D(y_hat) D_out_real = None if c.use_feat_match_loss: with torch.no_grad(): - D_out_real = model_D(in_real_D) + D_out_real = model_D(y_G) # format D outputs if isinstance(D_out_fake, tuple): @@ -162,7 +159,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, # compute losses loss_G_dict = criterion_G(y_hat, y_G, scores_fake, feats_fake, - feats_real) + feats_real, y_hat_sub, y_G_sub) loss_G = loss_G_dict['G_loss'] # optimizer generator @@ -272,12 +269,12 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, model_losses=loss_dict) # compute spectrograms - figures = plot_results(in_fake_D, in_real_D, ap, global_step, + figures = plot_results(y_hat, y_G, ap, global_step, 'train') tb_logger.tb_train_figures(global_step, figures) # Sample audio - sample_voice = in_fake_D[0].squeeze(0).detach().cpu().numpy() + sample_voice = y_hat[0].squeeze(0).detach().cpu().numpy() tb_logger.tb_train_audios(global_step, {'train/audio': sample_voice}, c.audio["sample_rate"]) @@ -320,23 +317,20 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): # generator pass y_hat = model_G(c_G) - - in_real_D = y_hat - in_fake_D = y_G + y_hat_sub = None + y_G_sub = None # PQMF formatting if y_hat.shape[1] > 1: - in_real_D = y_G - in_fake_D = model_G.pqmf_synthesis(y_hat) - y_G = model_G.pqmf_analysis(y_G) - y_hat = y_hat.view(-1, 1, y_hat.shape[2]) - y_G = y_G.view(-1, 1, y_G.shape[2]) + y_hat_sub = y_hat + y_hat = model_G.pqmf_synthesis(y_hat) + y_G_sub = model_G.pqmf_analysis(y_G) - D_out_fake = model_D(in_fake_D) + D_out_fake = model_D(y_hat) D_out_real = None if c.use_feat_match_loss: with torch.no_grad(): - D_out_real = model_D(in_real_D) + D_out_real = model_D(y_G) # format D outputs if isinstance(D_out_fake, tuple): @@ -350,7 +344,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): # compute losses loss_G_dict = criterion_G(y_hat, y_G, scores_fake, feats_fake, - feats_real) + feats_real, y_hat_sub, y_G_sub) loss_dict = dict() for key, value in loss_G_dict.items(): @@ -364,7 +358,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): for key, value in loss_G_dict.items(): update_eval_values['avg_' + key] = value.item() update_eval_values['avg_loader_time'] = loader_time - update_eval_values['avg_step_time'] = step_time + update_eval_values['avgP_step_time'] = step_time keep_avg.update_values(update_eval_values) # print eval stats @@ -372,18 +366,19 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) # compute spectrograms - figures = plot_results(in_fake_D, in_real_D, ap, global_step, 'eval') + figures = plot_results(y_hat, y_G, ap, global_step, 'eval') tb_logger.tb_eval_figures(global_step, figures) # Sample audio - sample_voice = in_fake_D[0].squeeze(0).detach().cpu().numpy() + sample_voice = y_hat[0].squeeze(0).detach().cpu().numpy() tb_logger.tb_eval_audios(global_step, {'eval/audio': sample_voice}, c.audio["sample_rate"]) # synthesize a full voice data_loader.return_segments = False - tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) + tb_logger.tb_evals_stats(global_step, keep_avg.avg_values) + return keep_avg.avg_values From fd2d0dab41764890dda0aadbad782eda29005d93 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 17:09:05 +0200 Subject: [PATCH 14/68] bug fix --- vocoder/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vocoder/train.py b/vocoder/train.py index 6341bb6c..398051a9 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -377,7 +377,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): # synthesize a full voice data_loader.return_segments = False - tb_logger.tb_evals_stats(global_step, keep_avg.avg_values) + tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) return keep_avg.avg_values From 808da15e7d50fd014687664c7664b989405a9d85 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 17:28:52 +0200 Subject: [PATCH 15/68] update travis script --- .travis/script | 4 +++- requirements_tests.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis/script b/.travis/script index ca6f4cd3..278ba4f2 100755 --- a/.travis/script +++ b/.travis/script @@ -12,7 +12,9 @@ fi if [[ "$TEST_SUITE" == "unittest" ]]; then # Run tests on all pushes pushd tts_namespace - python -m unittest + nosetests speaker_encoder.tests --nocapture + nosetests vocoder.tests --nocapture + nosetests tests --nocapture popd # Test server package ./tests/test_server_package.sh diff --git a/requirements_tests.txt b/requirements_tests.txt index 1e0615b2..243aa3a6 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -13,3 +13,4 @@ tqdm soundfile phonemizer bokeh==1.4.0 +nose From af516aff7fd6e8ee5a182af92507f6863b6b4783 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 18:54:03 +0200 Subject: [PATCH 16/68] cardboard config update --- .cardboardlint.yml | 3 +++ .pylintrc | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.cardboardlint.yml b/.cardboardlint.yml index 464ea733..4a115a37 100644 --- a/.cardboardlint.yml +++ b/.cardboardlint.yml @@ -1,2 +1,5 @@ linters: - pylint: + # pylintrc: pylintrc + filefilter: ['- test_*.py', '+ *.py', '- *.npy'] + # exclude: \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index b6e04944..a78b521e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -157,7 +157,8 @@ disable=missing-docstring, xreadlines-attribute, deprecated-sys-function, exception-escape, - comprehension-escape + comprehension-escape, + duplicate-code # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 054c1fce39bd89b8ce2194de641cb4e2c63f919b Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 18:54:29 +0200 Subject: [PATCH 17/68] travis script for nose --- .travis/script | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis/script b/.travis/script index 278ba4f2..ce886fc2 100755 --- a/.travis/script +++ b/.travis/script @@ -12,9 +12,9 @@ fi if [[ "$TEST_SUITE" == "unittest" ]]; then # Run tests on all pushes pushd tts_namespace - nosetests speaker_encoder.tests --nocapture - nosetests vocoder.tests --nocapture - nosetests tests --nocapture + nosetests TTS.speaker_encoder.tests --nocapture + nosetests TTS.vocoder.tests --nocapture + nosetests TTS.tests --nocapture popd # Test server package ./tests/test_server_package.sh From e051dd41b18fed43a71c9d6b7280548184fd3341 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 18:58:10 +0200 Subject: [PATCH 18/68] Ton of linter fixes --- train.py | 12 +-- utils/training.py | 11 +++ vocoder/datasets/gan_dataset.py | 34 ++----- vocoder/layers/losses.py | 35 ++----- vocoder/layers/melgan.py | 3 - vocoder/layers/pqmf.py | 3 + vocoder/layers/pqmf2.py | 127 ------------------------- vocoder/models/melgan_discriminator.py | 4 +- vocoder/models/melgan_generator.py | 2 - vocoder/tests/test_datasets.py | 22 ++--- vocoder/tests/test_losses.py | 3 +- vocoder/tests/test_melgan_generator.py | 1 - vocoder/tests/test_pqmf.py | 7 +- vocoder/train.py | 24 ++--- vocoder/utils/console_logger.py | 2 +- vocoder/utils/io.py | 1 - 16 files changed, 60 insertions(+), 231 deletions(-) delete mode 100644 vocoder/layers/pqmf2.py diff --git a/train.py b/train.py index 82f91cd5..5a345b59 100644 --- a/train.py +++ b/train.py @@ -20,7 +20,8 @@ from TTS.utils.generic_utils import (count_parameters, create_experiment_folder, from TTS.utils.io import (save_best_model, save_checkpoint, load_config, copy_config_file) from TTS.utils.training import (NoamLR, check_update, adam_weight_decay, - gradual_training_scheduler, set_weight_decay) + gradual_training_scheduler, set_weight_decay, + setup_torch_training_env) from TTS.utils.tensorboard_logger import TensorboardLogger from TTS.utils.console_logger import ConsoleLogger from TTS.utils.speakers import load_speaker_mapping, save_speaker_mapping, \ @@ -32,13 +33,8 @@ from TTS.datasets.preprocess import load_meta_data from TTS.utils.radam import RAdam from TTS.utils.measures import alignment_diagonal_score -torch.backends.cudnn.enabled = True -torch.backends.cudnn.benchmark = False -torch.manual_seed(54321) -use_cuda = torch.cuda.is_available() -num_gpus = torch.cuda.device_count() -print(" > Using CUDA: ", use_cuda) -print(" > Number of GPUs: ", num_gpus) + +use_cuda, num_gpus = setup_torch_training_env(True, False) def setup_loader(ap, r, is_val=False, verbose=False): diff --git a/utils/training.py b/utils/training.py index 5ce7948b..9046f9e0 100644 --- a/utils/training.py +++ b/utils/training.py @@ -2,6 +2,17 @@ import torch import numpy as np +def setup_torch_training_env(cudnn_enable, cudnn_benchmark): + torch.backends.cudnn.enabled = cudnn_enable + torch.backends.cudnn.benchmark = cudnn_benchmark + torch.manual_seed(54321) + use_cuda = torch.cuda.is_available() + num_gpus = torch.cuda.device_count() + print(" > Using CUDA: ", use_cuda) + print(" > Number of GPUs: ", num_gpus) + return use_cuda, num_gpus + + def check_update(model, grad_clip, ignore_stopnet=False): r'''Check model gradient against unexpected jumps and failures''' skip_flag = False diff --git a/vocoder/datasets/gan_dataset.py b/vocoder/datasets/gan_dataset.py index 10d36cab..5e410151 100644 --- a/vocoder/datasets/gan_dataset.py +++ b/vocoder/datasets/gan_dataset.py @@ -3,29 +3,10 @@ import glob import torch import random import numpy as np -from torch.utils.data import Dataset, DataLoader +from torch.utils.data import Dataset from multiprocessing import Manager -def create_dataloader(hp, args, train): - dataset = MelFromDisk(hp, args, train) - - if train: - return DataLoader(dataset=dataset, - batch_size=hp.train.batch_size, - shuffle=True, - num_workers=hp.train.num_workers, - pin_memory=True, - drop_last=True) - else: - return DataLoader(dataset=dataset, - batch_size=1, - shuffle=False, - num_workers=hp.train.num_workers, - pin_memory=True, - drop_last=False) - - class GANDataset(Dataset): """ GAN Dataset searchs for all the wav files under root path @@ -55,26 +36,26 @@ class GANDataset(Dataset): self.return_segments = return_segments self.use_cache = use_cache self.use_noise_augment = use_noise_augment + self.verbose = verbose assert seq_len % hop_len == 0, " [!] seq_len has to be a multiple of hop_len." self.feat_frame_len = seq_len // hop_len + (2 * conv_pad) # map G and D instances - self.G_to_D_mappings = [i for i in range(len(self.item_list))] + self.G_to_D_mappings = list(range(len(self.item_list))) self.shuffle_mapping() # cache acoustic features if use_cache: self.create_feature_cache() - - def create_feature_cache(self): self.manager = Manager() self.cache = self.manager.list() self.cache += [None for _ in range(len(self.item_list))] - def find_wav_files(self, path): + @staticmethod + def find_wav_files(path): return glob.glob(os.path.join(path, '**', '*.wav'), recursive=True) def __len__(self): @@ -88,9 +69,8 @@ class GANDataset(Dataset): item1 = self.load_item(idx) item2 = self.load_item(idx2) return item1, item2 - else: - item1 = self.load_item(idx) - return item1 + item1 = self.load_item(idx) + return item1 def shuffle_mapping(self): random.shuffle(self.G_to_D_mappings) diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index e242d209..1c3d4442 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -51,13 +51,13 @@ class STFTLoss(nn.Module): class MultiScaleSTFTLoss(torch.nn.Module): def __init__(self, - n_ffts=[1024, 2048, 512], - hop_lengths=[120, 240, 50], - win_lengths=[600, 1200, 240]): + n_ffts=(1024, 2048, 512), + hop_lengths=(120, 240, 50), + win_lengths=(600, 1200, 240)): super(MultiScaleSTFTLoss, self).__init__() self.loss_funcs = torch.nn.ModuleList() - for idx in range(len(n_ffts)): - self.loss_funcs.append(STFTLoss(n_ffts[idx], hop_lengths[idx], win_lengths[idx])) + for n_fft, hop_length, win_length in zip(n_ffts, hop_lengths, win_lengths): + self.loss_funcs.append(STFTLoss(n_fft, hop_length, win_length)) def forward(self, y_hat, y): N = len(self.loss_funcs) @@ -81,20 +81,14 @@ class MultiScaleSubbandSTFTLoss(MultiScaleSTFTLoss): class MSEGLoss(nn.Module): """ Mean Squared Generator Loss """ - def __init__(self,): - super(MSEGLoss, self).__init__() - - def forward(self, score_fake, ): + def forward(self, score_fake): loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) return loss_fake class HingeGLoss(nn.Module): """ Hinge Discriminator Loss """ - def __init__(self,): - super(HingeGLoss, self).__init__() - - def forward(self, score_fake, score_real): + def forward(self, score_fake): loss_fake = torch.mean(F.relu(1. + score_fake)) return loss_fake @@ -106,9 +100,6 @@ class HingeGLoss(nn.Module): class MSEDLoss(nn.Module): """ Mean Squared Discriminator Loss """ - def __init__(self,): - super(MSEDLoss, self).__init__() - def forward(self, score_fake, score_real): loss_real = torch.mean(torch.sum(torch.pow(score_real - 1.0, 2), dim=[1, 2])) loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) @@ -118,9 +109,6 @@ class MSEDLoss(nn.Module): class HingeDLoss(nn.Module): """ Hinge Discriminator Loss """ - def __init__(self,): - super(HingeDLoss, self).__init__() - def forward(self, score_fake, score_real): loss_real = torch.mean(F.relu(1. - score_real)) loss_fake = torch.mean(F.relu(1. + score_fake)) @@ -129,13 +117,10 @@ class HingeDLoss(nn.Module): class MelganFeatureLoss(nn.Module): - def __init__(self, ): - super(MelganFeatureLoss, self).__init__() - def forward(self, fake_feats, real_feats): loss_feats = 0 for fake_feat, real_feat in zip(fake_feats, real_feats): - loss_feats += hp.model.feat_match * torch.mean(torch.abs(fake_feat - real_feat)) + loss_feats += torch.mean(torch.abs(fake_feat - real_feat)) return loss_feats @@ -147,7 +132,7 @@ class MelganFeatureLoss(nn.Module): class GeneratorLoss(nn.Module): def __init__(self, C): super(GeneratorLoss, self).__init__() - assert C.use_mse_gan_loss and C.use_hinge_gan_loss == False,\ + assert not(C.use_mse_gan_loss and C.use_hinge_gan_loss),\ " [!] Cannot use HingeGANLoss and MSEGANLoss together." self.use_stft_loss = C.use_stft_loss @@ -228,7 +213,7 @@ class GeneratorLoss(nn.Module): class DiscriminatorLoss(nn.Module): def __init__(self, C): super(DiscriminatorLoss, self).__init__() - assert C.use_mse_gan_loss and C.use_hinge_gan_loss == False,\ + assert not(C.use_mse_gan_loss and C.use_hinge_gan_loss),\ " [!] Cannot use HingeGANLoss and MSEGANLoss together." self.use_mse_gan_loss = C.use_mse_gan_loss diff --git a/vocoder/layers/melgan.py b/vocoder/layers/melgan.py index cda0413c..61f7a96b 100644 --- a/vocoder/layers/melgan.py +++ b/vocoder/layers/melgan.py @@ -1,7 +1,4 @@ -import numpy as np -import torch from torch import nn -from torch.nn import functional as F from torch.nn.utils import weight_norm diff --git a/vocoder/layers/pqmf.py b/vocoder/layers/pqmf.py index f438ea00..3985aad4 100644 --- a/vocoder/layers/pqmf.py +++ b/vocoder/layers/pqmf.py @@ -44,6 +44,9 @@ class PQMF(torch.nn.Module): self.pad_fn = torch.nn.ConstantPad1d(taps // 2, 0.0) + def forward(self, x): + return self.analysis(x) + def analysis(self, x): return F.conv1d(x, self.H, padding=self.taps // 2, stride=self.N) diff --git a/vocoder/layers/pqmf2.py b/vocoder/layers/pqmf2.py deleted file mode 100644 index 4cffb819..00000000 --- a/vocoder/layers/pqmf2.py +++ /dev/null @@ -1,127 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright 2020 Tomoki Hayashi -# MIT License (https://opensource.org/licenses/MIT) - -"""Pseudo QMF modules.""" - -import numpy as np -import torch -import torch.nn.functional as F - -from scipy.signal import kaiser - - -def design_prototype_filter(taps=62, cutoff_ratio=0.15, beta=9.0): - """Design prototype filter for PQMF. - - This method is based on `A Kaiser window approach for the design of prototype - filters of cosine modulated filterbanks`_. - - Args: - taps (int): The number of filter taps. - cutoff_ratio (float): Cut-off frequency ratio. - beta (float): Beta coefficient for kaiser window. - - Returns: - ndarray: Impluse response of prototype filter (taps + 1,). - - .. _`A Kaiser window approach for the design of prototype filters of cosine modulated filterbanks`: - https://ieeexplore.ieee.org/abstract/document/681427 - - """ - # check the arguments are valid - assert taps % 2 == 0, "The number of taps mush be even number." - assert 0.0 < cutoff_ratio < 1.0, "Cutoff ratio must be > 0.0 and < 1.0." - - # make initial filter - omega_c = np.pi * cutoff_ratio - with np.errstate(invalid='ignore'): - h_i = np.sin(omega_c * (np.arange(taps + 1) - 0.5 * taps)) \ - / (np.pi * (np.arange(taps + 1) - 0.5 * taps)) - h_i[taps // 2] = np.cos(0) * cutoff_ratio # fix nan due to indeterminate form - - # apply kaiser window - w = kaiser(taps + 1, beta) - h = h_i * w - - return h - - -class PQMF(torch.nn.Module): - """PQMF module. - - This module is based on `Near-perfect-reconstruction pseudo-QMF banks`_. - - .. _`Near-perfect-reconstruction pseudo-QMF banks`: - https://ieeexplore.ieee.org/document/258122 - - """ - - def __init__(self, subbands=4, taps=62, cutoff_ratio=0.15, beta=9.0): - """Initilize PQMF module. - - Args: - subbands (int): The number of subbands. - taps (int): The number of filter taps. - cutoff_ratio (float): Cut-off frequency ratio. - beta (float): Beta coefficient for kaiser window. - - """ - super(PQMF, self).__init__() - - # define filter coefficient - h_proto = design_prototype_filter(taps, cutoff_ratio, beta) - h_analysis = np.zeros((subbands, len(h_proto))) - h_synthesis = np.zeros((subbands, len(h_proto))) - for k in range(subbands): - h_analysis[k] = 2 * h_proto * np.cos((2 * k + 1) * (np.pi / (2 * subbands)) * (np.arange(taps + 1) - ((taps - 1) / 2)) + (-1) ** k * np.pi / 4) - h_synthesis[k] = 2 * h_proto * np.cos((2 * k + 1) * (np.pi / (2 * subbands)) * (np.arange(taps + 1) - ((taps - 1) / 2)) - (-1) ** k * np.pi / 4) - - # convert to tensor - analysis_filter = torch.from_numpy(h_analysis).float().unsqueeze(1) - synthesis_filter = torch.from_numpy(h_synthesis).float().unsqueeze(0) - - # register coefficients as beffer - self.register_buffer("analysis_filter", analysis_filter) - self.register_buffer("synthesis_filter", synthesis_filter) - - # filter for downsampling & upsampling - updown_filter = torch.zeros((subbands, subbands, subbands)).float() - for k in range(subbands): - updown_filter[k, k, 0] = 1.0 - self.register_buffer("updown_filter", updown_filter) - self.subbands = subbands - - # keep padding info - self.pad_fn = torch.nn.ConstantPad1d(taps // 2, 0.0) - - def analysis(self, x): - """Analysis with PQMF. - - Args: - x (Tensor): Input tensor (B, 1, T). - - Returns: - Tensor: Output tensor (B, subbands, T // subbands). - - """ - x = F.conv1d(self.pad_fn(x), self.analysis_filter) - return F.conv1d(x, self.updown_filter, stride=self.subbands) - - def synthesis(self, x): - """Synthesis with PQMF. - - Args: - x (Tensor): Input tensor (B, subbands, T // subbands). - - Returns: - Tensor: Output tensor (B, 1, T). - - """ - # NOTE(kan-bayashi): Power will be dreased so here multipy by # subbands. - # Not sure this is the correct way, it is better to check again. - # TODO(kan-bayashi): Understand the reconstruction procedure - x = F.conv_transpose1d(x, self.updown_filter * self.subbands, stride=self.subbands) - x = F.conv1d(self.pad_fn(x), self.synthesis_filter) - return x diff --git a/vocoder/models/melgan_discriminator.py b/vocoder/models/melgan_discriminator.py index 55d3f585..3847babb 100644 --- a/vocoder/models/melgan_discriminator.py +++ b/vocoder/models/melgan_discriminator.py @@ -1,7 +1,5 @@ import numpy as np -import torch from torch import nn -from torch.nn import functional as F from torch.nn.utils import weight_norm @@ -32,7 +30,7 @@ class MelganDiscriminator(nn.Module): # downsampling layers layer_in_channels = base_channels - for idx, downsample_factor in enumerate(downsample_factors): + for downsample_factor in downsample_factors: layer_out_channels = min(layer_in_channels * downsample_factor, max_channels) layer_kernel_size = downsample_factor * 10 + 1 diff --git a/vocoder/models/melgan_generator.py b/vocoder/models/melgan_generator.py index 2d266f29..1e47816d 100644 --- a/vocoder/models/melgan_generator.py +++ b/vocoder/models/melgan_generator.py @@ -1,7 +1,5 @@ -import math import torch from torch import nn -from torch.nn import functional as F from torch.nn.utils import weight_norm from TTS.vocoder.layers.melgan import ResidualStack diff --git a/vocoder/tests/test_datasets.py b/vocoder/tests/test_datasets.py index 3d6280f0..5d409b3f 100644 --- a/vocoder/tests/test_datasets.py +++ b/vocoder/tests/test_datasets.py @@ -22,7 +22,7 @@ ok_ljspeech = os.path.exists(test_data_path) def gan_dataset_case(batch_size, seq_len, hop_len, conv_pad, return_segments, use_noise_augment, use_cache, num_workers): ''' run dataloader with given parameters and check conditions ''' ap = AudioProcessor(**C.audio) - eval_items, train_items = load_wav_data(test_data_path, 10) + _, train_items = load_wav_data(test_data_path, 10) dataset = GANDataset(ap, train_items, seq_len=seq_len, @@ -44,9 +44,9 @@ def gan_dataset_case(batch_size, seq_len, hop_len, conv_pad, return_segments, us # return random segments or return the whole audio if return_segments: - for item1, item2 in loader: + for item1, _ in loader: feat1, wav1 = item1 - feat2, wav2 = item2 + # feat2, wav2 = item2 expected_feat_shape = (batch_size, ap.num_mels, seq_len // hop_len + conv_pad * 2) # check shapes @@ -80,15 +80,15 @@ def gan_dataset_case(batch_size, seq_len, hop_len, conv_pad, return_segments, us def test_parametrized_gan_dataset(): ''' test dataloader with different parameters ''' params = [ - [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], - [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 4], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, True, 0], + [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], + [32, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 4], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, True, 0], [1, C.audio['hop_length'], C.audio['hop_length'], 0, True, True, True, 0], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, True, True, True, 0], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, True, True, 0], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, False, 0], - [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, False, False, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 2, True, True, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, True, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, False, True, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, True, True, False, 0], + [1, C.audio['hop_length'] * 10, C.audio['hop_length'], 0, False, False, False, 0], ] for param in params: print(param) diff --git a/vocoder/tests/test_losses.py b/vocoder/tests/test_losses.py index 83314832..44a1321d 100644 --- a/vocoder/tests/test_losses.py +++ b/vocoder/tests/test_losses.py @@ -1,5 +1,4 @@ import os -import unittest import torch from TTS.vocoder.layers.losses import TorchSTFT, STFTLoss, MultiScaleSTFTLoss @@ -24,7 +23,7 @@ def test_torch_stft(): torch_stft = TorchSTFT(ap.n_fft, ap.hop_length, ap.win_length) # librosa stft wav = ap.load_wav(WAV_FILE) - M_librosa = abs(ap._stft(wav)) + M_librosa = abs(ap._stft(wav)) # pylint: disable=protected-access # torch stft wav = torch.from_numpy(wav[None, :]).float() M_torch = torch_stft(wav) diff --git a/vocoder/tests/test_melgan_generator.py b/vocoder/tests/test_melgan_generator.py index e9c4ad60..e03da31b 100644 --- a/vocoder/tests/test_melgan_generator.py +++ b/vocoder/tests/test_melgan_generator.py @@ -1,5 +1,4 @@ import numpy as np -import unittest import torch from TTS.vocoder.models.melgan_generator import MelganGenerator diff --git a/vocoder/tests/test_pqmf.py b/vocoder/tests/test_pqmf.py index 8444fc5a..a26bdd59 100644 --- a/vocoder/tests/test_pqmf.py +++ b/vocoder/tests/test_pqmf.py @@ -1,16 +1,11 @@ import os import torch -import unittest -import numpy as np import soundfile as sf from librosa.core import load -from TTS.tests import get_tests_path, get_tests_input_path, get_tests_output_path -from TTS.utils.audio import AudioProcessor -from TTS.utils.io import load_config +from TTS.tests import get_tests_path, get_tests_input_path from TTS.vocoder.layers.pqmf import PQMF -from TTS.vocoder.layers.pqmf2 import PQMF as PQMF2 TESTS_PATH = get_tests_path() diff --git a/vocoder/train.py b/vocoder/train.py index 398051a9..d401e72e 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -17,7 +17,7 @@ from TTS.utils.generic_utils import (KeepAverage, count_parameters, from TTS.utils.io import copy_config_file, load_config from TTS.utils.radam import RAdam from TTS.utils.tensorboard_logger import TensorboardLogger -from TTS.utils.training import NoamLR +from TTS.utils.training import setup_torch_training_env, NoamLR from TTS.vocoder.datasets.gan_dataset import GANDataset from TTS.vocoder.datasets.preprocess import load_wav_data # from distribute import (DistributedSampler, apply_gradient_allreduce, @@ -29,13 +29,8 @@ from TTS.vocoder.utils.generic_utils import (check_config, plot_results, setup_discriminator, setup_generator) -torch.backends.cudnn.enabled = True -torch.backends.cudnn.benchmark = True -torch.manual_seed(54321) -use_cuda = torch.cuda.is_available() -num_gpus = torch.cuda.device_count() -print(" > Using CUDA: ", use_cuda) -print(" > Number of GPUs: ", num_gpus) + +use_cuda, num_gpus = setup_torch_training_env(True, True) def setup_loader(ap, is_val=False, verbose=False): @@ -49,10 +44,11 @@ def setup_loader(ap, is_val=False, verbose=False): pad_short=c.pad_short, conv_pad=c.conv_pad, is_training=not is_val, - return_segments=False if is_val else True, + return_segments=not is_val, use_noise_augment=c.use_noise_augment, use_cache=c.use_cache, verbose=verbose) + dataset.shuffle_mapping() # sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader(dataset, batch_size=1 if is_val else c.batch_size, @@ -81,11 +77,11 @@ def format_data(data): return c_G, x_G, c_D, x_D # return a whole audio segment - c, x = data + co, x = data if use_cuda: - c = c.cuda(non_blocking=True) + co = co.cuda(non_blocking=True) x = x.cuda(non_blocking=True) - return c, x + return co, x, None, None def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, @@ -306,7 +302,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): start_time = time.time() # format data - c_G, y_G = format_data(data) + c_G, y_G, _, _ = format_data(data) loader_time = time.time() - end_time global_step += 1 @@ -416,7 +412,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen.load_state_dict(checkpoint['optimizer']) model_disc.load_state_dict(checkpoint['model_disc']) optimizer_disc.load_state_dict(checkpoint['optimizer_disc']) - except: + except KeyError: print(" > Partial model initialization.") model_dict = model_gen.state_dict() model_dict = set_init_dict(model_dict, checkpoint['model'], c) diff --git a/vocoder/utils/console_logger.py b/vocoder/utils/console_logger.py index 4fe132bb..ff7edfd0 100644 --- a/vocoder/utils/console_logger.py +++ b/vocoder/utils/console_logger.py @@ -91,7 +91,7 @@ class ConsoleLogger(): sign = '' elif diff > 0: color = tcolors.FAIL - sing = '+' + sign = '+' log_text += "{}{}:{} {:.5f} {}({}{:.5f})\n".format(indent, key, color, value, tcolors.ENDC, sign, diff) self.old_eval_loss_dict = avg_loss_dict print(log_text, flush=True) diff --git a/vocoder/utils/io.py b/vocoder/utils/io.py index 9526b9db..b6ea7484 100644 --- a/vocoder/utils/io.py +++ b/vocoder/utils/io.py @@ -2,7 +2,6 @@ import os import torch import datetime - def save_model(model, optimizer, model_disc, optimizer_disc, current_step, epoch, output_path, **kwargs): model_state = model.state_dict() From b25af49c34b161cf0d2d7e6887af35a811bde33d Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 2 Jun 2020 23:58:18 +0200 Subject: [PATCH 19/68] espeak install travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 645f9861..83ba25a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,10 @@ language: python git: quiet: true +before_install: + - sudo apt-get update + - sudo apt-get -y install espeak + matrix: include: - name: "Lint check" From 1e017f866abf5d1249d19ec77a3a91a7b7ff227d Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 3 Jun 2020 12:15:48 +0200 Subject: [PATCH 20/68] reset loss weights --- vocoder/configs/melgan_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json index 4aeba82e..bdfdf88a 100644 --- a/vocoder/configs/melgan_config.json +++ b/vocoder/configs/melgan_config.json @@ -54,11 +54,11 @@ "use_hinge_gan_loss": false, "use_feat_match_loss": false, // use only with melgan discriminators - "stft_loss_weight": 1, - "subband_stft_loss_weight": 1, + "stft_loss_weight": 0.5, + "subband_stft_loss_weight": 0.5, "mse_gan_loss_weight": 2.5, "hinge_gan_loss_weight": 2.5, - "feat_match_loss_weight": 10.0, + "feat_match_loss_weight": 25, "stft_loss_params": { "n_ffts": [1024, 2048, 512], From 20a9d51cccbe4e995aa34d4965bde8dcd25b8f6b Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 3 Jun 2020 12:16:08 +0200 Subject: [PATCH 21/68] correct loss normalization and function refactoring --- vocoder/layers/losses.py | 141 +++++++++++++++++++++------------------ 1 file changed, 77 insertions(+), 64 deletions(-) diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index 1c3d4442..fb4e85d4 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -6,6 +6,7 @@ from torch.nn import functional as F class TorchSTFT(): def __init__(self, n_fft, hop_length, win_length, window='hann_window'): + """ Torch based STFT operation """ self.n_fft = n_fft self.hop_length = hop_length self.win_length = win_length @@ -33,6 +34,7 @@ class TorchSTFT(): class STFTLoss(nn.Module): + """ Single scale STFT Loss """ def __init__(self, n_fft, hop_length, win_length): super(STFTLoss, self).__init__() self.n_fft = n_fft @@ -50,6 +52,7 @@ class STFTLoss(nn.Module): return loss_mag, loss_sc class MultiScaleSTFTLoss(torch.nn.Module): + """ Multi scale STFT loss """ def __init__(self, n_ffts=(1024, 2048, 512), hop_lengths=(120, 240, 50), @@ -73,6 +76,7 @@ class MultiScaleSTFTLoss(torch.nn.Module): class MultiScaleSubbandSTFTLoss(MultiScaleSTFTLoss): + """ Multiscale STFT loss for multi band model outputs """ def forward(self, y_hat, y): y_hat = y_hat.view(-1, 1, y_hat.shape[2]) y = y.view(-1, 1, y.shape[2]) @@ -121,16 +125,62 @@ class MelganFeatureLoss(nn.Module): loss_feats = 0 for fake_feat, real_feat in zip(fake_feats, real_feats): loss_feats += torch.mean(torch.abs(fake_feat - real_feat)) + loss_feats /= len(fake_feats) + len(real_feats) return loss_feats -################################## +##################################### # LOSS WRAPPERS +##################################### + + +def _apply_G_adv_loss(scores_fake, loss_func): + """ Compute G adversarial loss function + and normalize values """ + adv_loss = 0 + if isinstance(scores_fake, list): + for score_fake in scores_fake: + fake_loss = loss_func(score_fake) + adv_loss += fake_loss + adv_loss /= len(scores_fake) + else: + fake_loss = loss_func(scores_fake) + adv_loss = fake_loss + return adv_loss + + +def _apply_D_loss(scores_fake, scores_real, loss_func): + """ Compute D loss func and normalize loss values """ + loss = 0 + real_loss = 0 + fake_loss = 0 + if isinstance(scores_fake, list): + # multi-scale loss + for score_fake, score_real in zip(scores_fake, scores_real): + total_loss, real_loss, fake_loss = loss_func(score_fake, score_real) + loss += total_loss + real_loss += real_loss + fake_loss += fake_loss + # normalize loss values with number of scales + loss /= len(scores_fake) + real_loss /= len(scores_real) + fake_loss /= len(scores_fake) + else: + # single scale loss + total_loss, real_loss, fake_loss = loss_func(scores_fake, scores_real) + loss = total_loss + return loss, real_loss, fake_loss + + +################################## +# MODEL LOSSES ################################## class GeneratorLoss(nn.Module): def __init__(self, C): + """ Compute Generator Loss values depending on training + configuration """ super(GeneratorLoss, self).__init__() assert not(C.use_mse_gan_loss and C.use_hinge_gan_loss),\ " [!] Cannot use HingeGANLoss and MSEGANLoss together." @@ -159,7 +209,8 @@ class GeneratorLoss(nn.Module): self.feat_match_loss = MelganFeatureLoss() def forward(self, y_hat=None, y=None, scores_fake=None, feats_fake=None, feats_real=None, y_hat_sub=None, y_sub=None): - loss = 0 + gen_loss = 0 + adv_loss = 0 return_dict = {} # STFT Loss @@ -167,50 +218,41 @@ class GeneratorLoss(nn.Module): stft_loss_mg, stft_loss_sc = self.stft_loss(y_hat.squeeze(1), y.squeeze(1)) return_dict['G_stft_loss_mg'] = stft_loss_mg return_dict['G_stft_loss_sc'] = stft_loss_sc - loss += self.stft_loss_weight * (stft_loss_mg + stft_loss_sc) + gen_loss += self.stft_loss_weight * (stft_loss_mg + stft_loss_sc) # subband STFT Loss if self.use_subband_stft_loss: subband_stft_loss_mg, subband_stft_loss_sc = self.subband_stft_loss(y_hat_sub, y_sub) return_dict['G_subband_stft_loss_mg'] = subband_stft_loss_mg return_dict['G_subband_stft_loss_sc'] = subband_stft_loss_sc - loss += self.subband_stft_loss_weight * (subband_stft_loss_mg + subband_stft_loss_sc) + gen_loss += self.subband_stft_loss_weight * (subband_stft_loss_mg + subband_stft_loss_sc) - # Fake Losses + # multiscale MSE adversarial loss if self.use_mse_gan_loss and scores_fake is not None: - mse_fake_loss = 0 - if isinstance(scores_fake, list): - for score_fake in scores_fake: - fake_loss = self.mse_loss(score_fake) - mse_fake_loss += fake_loss - else: - fake_loss = self.mse_loss(scores_fake) - mse_fake_loss = fake_loss + mse_fake_loss = _apply_G_adv_loss(scores_fake, self.mse_loss) return_dict['G_mse_fake_loss'] = mse_fake_loss - loss += self.mse_gan_loss_weight * mse_fake_loss + adv_loss += self.mse_gan_loss_weight * mse_fake_loss + # multiscale Hinge adversarial loss if self.use_hinge_gan_loss and not scores_fake is not None: - hinge_fake_loss = 0 - if isinstance(scores_fake, list): - for score_fake in scores_fake: - fake_loss = self.hinge_loss(score_fake) - hinge_fake_loss += fake_loss - else: - fake_loss = self.hinge_loss(scores_fake) - hinge_fake_loss = fake_loss + hinge_fake_loss = _apply_G_adv_loss(scores_fake, self.hinge_loss) return_dict['G_hinge_fake_loss'] = hinge_fake_loss - loss += self.hinge_gan_loss_weight * hinge_fake_loss + adv_loss += self.hinge_gan_loss_weight * hinge_fake_loss # Feature Matching Loss if self.use_feat_match_loss and not feats_fake: feat_match_loss = self.feat_match_loss(feats_fake, feats_real) return_dict['G_feat_match_loss'] = feat_match_loss - loss += self.feat_match_loss_weight * feat_match_loss - return_dict['G_loss'] = loss + adv_loss += self.feat_match_loss_weight * feat_match_loss + return_dict['G_loss'] = gen_loss + adv_loss + return_dict['G_gen_loss'] = gen_loss + return_dict['G_adv_loss'] = adv_loss return return_dict class DiscriminatorLoss(nn.Module): + """ Compute Discriminator Loss values depending on training + configuration """ def __init__(self, C): super(DiscriminatorLoss, self).__init__() assert not(C.use_mse_gan_loss and C.use_hinge_gan_loss),\ @@ -219,9 +261,6 @@ class DiscriminatorLoss(nn.Module): self.use_mse_gan_loss = C.use_mse_gan_loss self.use_hinge_gan_loss = C.use_hinge_gan_loss - self.mse_gan_loss_weight = C.mse_gan_loss_weight - self.hinge_gan_loss_weight = C.hinge_gan_loss_weight - if C.use_mse_gan_loss: self.mse_loss = MSEDLoss() if C.use_hinge_gan_loss: @@ -232,44 +271,18 @@ class DiscriminatorLoss(nn.Module): return_dict = {} if self.use_mse_gan_loss: - mse_gan_loss = 0 - mse_gan_real_loss = 0 - mse_gan_fake_loss = 0 - if isinstance(scores_fake, list): - for score_fake, score_real in zip(scores_fake, scores_real): - total_loss, real_loss, fake_loss = self.mse_loss(score_fake, score_real) - mse_gan_loss += total_loss - mse_gan_real_loss += real_loss - mse_gan_fake_loss += fake_loss - else: - total_loss, real_loss, fake_loss = self.mse_loss(scores_fake, scores_real) - mse_gan_loss = total_loss - mse_gan_real_loss = real_loss - mse_gan_fake_loss = fake_loss - return_dict['D_mse_gan_loss'] = mse_gan_loss - return_dict['D_mse_gan_real_loss'] = mse_gan_real_loss - return_dict['D_mse_gan_fake_loss'] = mse_gan_fake_loss - loss += self.mse_gan_loss_weight * mse_gan_loss + mse_D_loss, mse_D_real_loss, mse_D_fake_loss = _apply_D_loss(scores_fake, scores_real, self.mse_loss) + return_dict['D_mse_gan_loss'] = mse_D_loss + return_dict['D_mse_gan_real_loss'] = mse_D_real_loss + return_dict['D_mse_gan_fake_loss'] = mse_D_fake_loss + loss += mse_D_loss if self.use_hinge_gan_loss: - hinge_gan_loss = 0 - hinge_gan_real_loss = 0 - hinge_gan_fake_loss = 0 - if isinstance(scores_fake, list): - for score_fake, score_real in zip(scores_fake, scores_real): - total_loss, real_loss, fake_loss = self.hinge_loss(score_fake, score_real) - hinge_gan_loss += total_loss - hinge_gan_real_loss += real_loss - hinge_gan_fake_loss += fake_loss - else: - total_loss, real_loss, fake_loss = self.hinge_loss(scores_fake, scores_real) - hinge_gan_loss = total_loss - hinge_gan_real_loss = real_loss - hinge_gan_fake_loss = fake_loss - return_dict['D_hinge_gan_loss'] = hinge_gan_loss - return_dict['D_hinge_gan_real_loss'] = hinge_gan_real_loss - return_dict['D_hinge_gan_fake_loss'] = hinge_gan_fake_loss - loss += self.hinge_gan_loss_weight * hinge_gan_loss + hinge_D_loss, hinge_D_real_loss, hinge_D_fake_loss = _apply_D_loss(scores_fake, scores_real, self.hinge_loss) + return_dict['D_hinge_gan_loss'] = hinge_D_loss + return_dict['D_hinge_gan_real_loss'] = hinge_D_real_loss + return_dict['D_hinge_gan_fake_loss'] = hinge_D_fake_loss + loss += hinge_D_loss return_dict['D_loss'] = loss return return_dict \ No newline at end of file From be861d7aa0ebfa0d926f579c4d9793314aac2808 Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 3 Jun 2020 12:33:53 +0200 Subject: [PATCH 22/68] suppress pylint no-self-use for loss layers --- vocoder/layers/losses.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index fb4e85d4..22e2fb54 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -77,6 +77,7 @@ class MultiScaleSTFTLoss(torch.nn.Module): class MultiScaleSubbandSTFTLoss(MultiScaleSTFTLoss): """ Multiscale STFT loss for multi band model outputs """ + # pylint: disable=no-self-use def forward(self, y_hat, y): y_hat = y_hat.view(-1, 1, y_hat.shape[2]) y = y.view(-1, 1, y.shape[2]) @@ -85,6 +86,7 @@ class MultiScaleSubbandSTFTLoss(MultiScaleSTFTLoss): class MSEGLoss(nn.Module): """ Mean Squared Generator Loss """ + # pylint: disable=no-self-use def forward(self, score_fake): loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) return loss_fake @@ -92,6 +94,7 @@ class MSEGLoss(nn.Module): class HingeGLoss(nn.Module): """ Hinge Discriminator Loss """ + # pylint: disable=no-self-use def forward(self, score_fake): loss_fake = torch.mean(F.relu(1. + score_fake)) return loss_fake @@ -104,6 +107,7 @@ class HingeGLoss(nn.Module): class MSEDLoss(nn.Module): """ Mean Squared Discriminator Loss """ + # pylint: disable=no-self-use def forward(self, score_fake, score_real): loss_real = torch.mean(torch.sum(torch.pow(score_real - 1.0, 2), dim=[1, 2])) loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) @@ -113,6 +117,7 @@ class MSEDLoss(nn.Module): class HingeDLoss(nn.Module): """ Hinge Discriminator Loss """ + # pylint: disable=no-self-use def forward(self, score_fake, score_real): loss_real = torch.mean(F.relu(1. - score_real)) loss_fake = torch.mean(F.relu(1. + score_fake)) @@ -121,6 +126,7 @@ class HingeDLoss(nn.Module): class MelganFeatureLoss(nn.Module): + # pylint: disable=no-self-use def forward(self, fake_feats, real_feats): loss_feats = 0 for fake_feat, real_feat in zip(fake_feats, real_feats): From ed466c8581058d5d9eeaffd237344270ba1328c7 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 02:58:25 +0200 Subject: [PATCH 23/68] update README.md --- README.md | 8 +++++--- vocoder/train.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7d9884b0..20799943 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,11 @@ If you are new, you can also find [here](http://www.erogol.com/text-speech-deep- [Details...](https://github.com/mozilla/TTS/wiki/Mean-Opinion-Score-Results) ## Features -- High performance Text2Speech models on Torch and Tensorflow 2.0. -- High performance Speaker Encoder to compute speaker embeddings efficiently. -- Integration with various Neural Vocoders (PWGAN, MelGAN, WaveRNN) +- High performance Deep Learning models for Text2Speech related tasks. + - Text2Speech models (Tacotron, Tacotron2). + - Speaker Encoder to compute speaker embeddings efficiently. + - Vocoder models (MelGAN, Multiband-MelGAN, GAN-TTS) +- Ability to convert Torch models to Tensorflow 2.0 for inference. - Released trained models. - Efficient training codes for PyTorch. (soon for Tensorflow 2.0) - Codes to convert Torch models to Tensorflow 2.0. diff --git a/vocoder/train.py b/vocoder/train.py index d401e72e..03e14f4a 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -52,7 +52,7 @@ def setup_loader(ap, is_val=False, verbose=False): # sampler = DistributedSampler(dataset) if num_gpus > 1 else None loader = DataLoader(dataset, batch_size=1 if is_val else c.batch_size, - shuffle=False, + shuffle=True, drop_last=False, sampler=None, num_workers=c.num_val_loader_workers From 2b361f0b50daed9e4555d7c09f371ba201564ed7 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 09:51:20 +0200 Subject: [PATCH 24/68] visualization fix --- vocoder/train.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vocoder/train.py b/vocoder/train.py index 03e14f4a..a75c6a12 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -120,11 +120,13 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, y_hat = model_G(c_G) y_hat_sub = None y_G_sub = None + y_hat_vis = y_hat # for visualization # PQMF formatting if y_hat.shape[1] > 1: y_hat_sub = y_hat y_hat = model_G.pqmf_synthesis(y_hat) + y_hat_vis = y_hat y_G_sub = model_G.pqmf_analysis(y_G) if global_step > c.steps_to_start_discriminator: @@ -265,12 +267,12 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, model_losses=loss_dict) # compute spectrograms - figures = plot_results(y_hat, y_G, ap, global_step, + figures = plot_results(y_hat_vis, y_G, ap, global_step, 'train') tb_logger.tb_train_figures(global_step, figures) # Sample audio - sample_voice = y_hat[0].squeeze(0).detach().cpu().numpy() + sample_voice = y_hat_vis[0].squeeze(0).detach().cpu().numpy() tb_logger.tb_train_audios(global_step, {'train/audio': sample_voice}, c.audio["sample_rate"]) From 1de68250407cb1b711d22fcd00018d912eb09849 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:26:30 +0200 Subject: [PATCH 25/68] tacotron abstract class --- models/tacotron_abstract.py | 160 ++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 models/tacotron_abstract.py diff --git a/models/tacotron_abstract.py b/models/tacotron_abstract.py new file mode 100644 index 00000000..db6cbdee --- /dev/null +++ b/models/tacotron_abstract.py @@ -0,0 +1,160 @@ +import copy +from abc import ABC, abstractmethod + +import torch +from torch import nn + +from TTS.layers.gst_layers import GST +from TTS.utils.generic_utils import sequence_mask + + +class TacotronAbstract(ABC, nn.Module): + def __init__(self, num_chars, + num_speakers, + r, + postnet_output_dim=80, + decoder_output_dim=80, + attn_type='original', + attn_win=False, + attn_norm="softmax", + prenet_type="original", + prenet_dropout=True, + forward_attn=False, + trans_agent=False, + forward_attn_mask=False, + location_attn=True, + attn_K=5, + separate_stopnet=True, + bidirectional_decoder=False, + double_decoder_consistency=False, + ddc_r=None, + gst=False): + """ Abstract Tacotron class """ + super().__init__() + self.r = r + self.decoder_output_dim = decoder_output_dim + self.postnet_output_dim = postnet_output_dim + self.gst = gst + self.num_speakers = num_speakers + self.bidirectional_decoder = bidirectional_decoder + self.double_decoder_consistency = double_decoder_consistency + self.ddc_r = ddc_r + + # layers + self.embedding = None + self.encoder = None + self.decoder = None + self.postnet = None + + # global style token + if self.gst: + gst_embedding_dim = None + self.gst_layer = None + + ############################# + # INIT FUNCTIONS + ############################# + + def _init_states(self): + self.speaker_embeddings = None + self.speaker_embeddings_projected = None + + def _init_backward_decoder(self): + self.decoder_backward = copy.deepcopy(self.decoder) + + def _init_coarse_decoder(self): + self.coarse_decoder = copy.deepcopy(self.decoder) + self.coarse_decoder.r_init = self.ddc_r + self.coarse_decoder.set_r(self.ddc_r) + + ############################# + # CORE FUNCTIONS + ############################# + + @abstractmethod + def forward(self): + pass + + @abstractmethod + def inference(self): + pass + + ############################# + # COMMON COMPUTE FUNCTIONS + ############################# + + def compute_masks(self, text_lengths, mel_lengths): + """Compute masks against sequence paddings.""" + # B x T_in_max (boolean) + device = text_lengths.device + input_mask = sequence_mask(text_lengths).to(device) + output_mask = None + if mel_lengths is not None: + max_len = mel_lengths.max() + r = self.decoder.r + max_len = max_len + (r - (max_len % r)) if max_len % r > 0 else max_len + output_mask = sequence_mask(mel_lengths, max_len=max_len).to(device) + return input_mask, output_mask + + def _backward_pass(self, mel_specs, encoder_outputs, mask): + """ Run backwards decoder """ + decoder_outputs_b, alignments_b, _ = self.decoder_backward( + encoder_outputs, torch.flip(mel_specs, dims=(1,)), mask, + self.speaker_embeddings_projected) + decoder_outputs_b = decoder_outputs_b.transpose(1, 2).contiguous() + return decoder_outputs_b, alignments_b + + def _coarse_decoder_pass(self, mel_specs, encoder_outputs, alignments, + input_mask): + """ Double Decoder Consistency """ + T = mel_specs.shape[1] + if T % self.coarse_decoder.r > 0: + padding_size = self.coarse_decoder.r - (T % self.coarse_decoder.r) + mel_specs = torch.nn.functional.pad(mel_specs, + (0, 0, 0, padding_size, 0, 0)) + decoder_outputs_backward, alignments_backward, _ = self.coarse_decoder( + encoder_outputs.detach(), mel_specs, input_mask) + scale_factor = self.decoder.r_init / self.decoder.r + alignments_backward = torch.nn.functional.interpolate( + alignments_backward.transpose(1, 2), + size=alignments.shape[1], + mode='nearest').transpose(1, 2) + decoder_outputs_backward = decoder_outputs_backward.transpose(1, 2) + decoder_outputs_backward = decoder_outputs_backward[:, :T, :] + return decoder_outputs_backward, alignments_backward + + ############################# + # EMBEDDING FUNCTIONS + ############################# + + def compute_speaker_embedding(self, speaker_ids): + """ Compute speaker embedding vectors """ + if hasattr(self, "speaker_embedding") and speaker_ids is None: + raise RuntimeError( + " [!] Model has speaker embedding layer but speaker_id is not provided" + ) + if hasattr(self, "speaker_embedding") and speaker_ids is not None: + self.speaker_embeddings = self.speaker_embedding(speaker_ids).unsqueeze(1) + if hasattr(self, "speaker_project_mel") and speaker_ids is not None: + self.speaker_embeddings_projected = self.speaker_project_mel( + self.speaker_embeddings).squeeze(1) + + def compute_gst(self, inputs, mel_specs): + """ Compute global style token """ + gst_outputs = self.gst_layer(mel_specs) + inputs = self._add_speaker_embedding(inputs, gst_outputs) + return inputs + + @staticmethod + def _add_speaker_embedding(outputs, speaker_embeddings): + speaker_embeddings_ = speaker_embeddings.expand( + outputs.size(0), outputs.size(1), -1) + outputs = outputs + speaker_embeddings_ + return outputs + + @staticmethod + def _concat_speaker_embedding(outputs, speaker_embeddings): + speaker_embeddings_ = speaker_embeddings.expand( + outputs.size(0), outputs.size(1), -1) + outputs = torch.cat([outputs, speaker_embeddings_], dim=-1) + return outputs From ef2369e0dac6eaa88489611c925bc929bc31571f Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:26:51 +0200 Subject: [PATCH 26/68] update README.md --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 20799943..f6ee1ade 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ If you are new, you can also find [here](http://www.erogol.com/text-speech-deep- [![](https://sourcerer.io/fame/erogol/erogol/TTS/images/0)](https://sourcerer.io/fame/erogol/erogol/TTS/links/0)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/1)](https://sourcerer.io/fame/erogol/erogol/TTS/links/1)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/2)](https://sourcerer.io/fame/erogol/erogol/TTS/links/2)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/3)](https://sourcerer.io/fame/erogol/erogol/TTS/links/3)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/4)](https://sourcerer.io/fame/erogol/erogol/TTS/links/4)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/5)](https://sourcerer.io/fame/erogol/erogol/TTS/links/5)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/6)](https://sourcerer.io/fame/erogol/erogol/TTS/links/6)[![](https://sourcerer.io/fame/erogol/erogol/TTS/images/7)](https://sourcerer.io/fame/erogol/erogol/TTS/links/7) -## TTS Performance +## TTS Performance

[Details...](https://github.com/mozilla/TTS/wiki/Mean-Opinion-Score-Results) @@ -19,8 +19,9 @@ If you are new, you can also find [here](http://www.erogol.com/text-speech-deep- ## Features - High performance Deep Learning models for Text2Speech related tasks. - Text2Speech models (Tacotron, Tacotron2). - - Speaker Encoder to compute speaker embeddings efficiently. + - Speaker Encoder to compute speaker embeddings efficiently. - Vocoder models (MelGAN, Multiband-MelGAN, GAN-TTS) +- Support for multi-speaker TTS training. - Ability to convert Torch models to Tensorflow 2.0 for inference. - Released trained models. - Efficient training codes for PyTorch. (soon for Tensorflow 2.0) @@ -86,7 +87,7 @@ Audio length is approximately 6 secs. ## Datasets and Data-Loading -TTS provides a generic dataloder easy to use for new datasets. You need to write an preprocessor function to integrate your own dataset.Check ```datasets/preprocess.py``` to see some examples. After the function, you need to set ```dataset``` field in ```config.json```. Do not forget other data related fields too. +TTS provides a generic dataloder easy to use for new datasets. You need to write an preprocessor function to integrate your own dataset.Check ```datasets/preprocess.py``` to see some examples. After the function, you need to set ```dataset``` field in ```config.json```. Do not forget other data related fields too. Some of the open-sourced datasets that we successfully applied TTS, are linked below. @@ -98,9 +99,9 @@ Some of the open-sourced datasets that we successfully applied TTS, are linked b - [Spanish](https://drive.google.com/file/d/1Sm_zyBo67XHkiFhcRSQ4YaHPYM0slO_e/view?usp=sharing) - thx! @carlfm01 ## Training and Fine-tuning LJ-Speech -Here you can find a [CoLab](https://gist.github.com/erogol/97516ad65b44dbddb8cd694953187c5b) notebook for a hands-on example, training LJSpeech. Or you can manually follow the guideline below. +Here you can find a [CoLab](https://gist.github.com/erogol/97516ad65b44dbddb8cd694953187c5b) notebook for a hands-on example, training LJSpeech. Or you can manually follow the guideline below. -To start with, split ```metadata.csv``` into train and validation subsets respectively ```metadata_train.csv``` and ```metadata_val.csv```. Note that for text-to-speech, validation performance might be misleading since the loss value does not directly measure the voice quality to the human ear and it also does not measure the attention module performance. Therefore, running the model with new sentences and listening to the results is the best way to go. +To start with, split ```metadata.csv``` into train and validation subsets respectively ```metadata_train.csv``` and ```metadata_val.csv```. Note that for text-to-speech, validation performance might be misleading since the loss value does not directly measure the voice quality to the human ear and it also does not measure the attention module performance. Therefore, running the model with new sentences and listening to the results is the best way to go. ``` shuf metadata.csv > metadata_shuf.csv @@ -139,10 +140,10 @@ cardboardlinter --refspec master ``` ## Collaborative Experimentation Guide -If you like to use TTS to try a new idea and like to share your experiments with the community, we urge you to use the following guideline for a better collaboration. +If you like to use TTS to try a new idea and like to share your experiments with the community, we urge you to use the following guideline for a better collaboration. (If you have an idea for better collaboration, let us know) - Create a new branch. -- Open an issue pointing your branch. +- Open an issue pointing your branch. - Explain your experiment. - Share your results as you proceed. (Tensorboard log files, audio results, visuals etc.) - Use LJSpeech dataset (for English) if you like to compare results with the released models. (It is the most open scalable dataset for quick experimentation) From 1ca078ae7e06ab9ccb79cf6022ad56052db84964 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:27:06 +0200 Subject: [PATCH 27/68] update model config.json with double decoder consistency --- config.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.json b/config.json index 32debf86..74021970 100644 --- a/config.json +++ b/config.json @@ -96,6 +96,8 @@ "transition_agent": false, // enable/disable transition agent of forward attention. "location_attn": true, // enable_disable location sensitive attention. It is enabled for TACOTRON by default. "bidirectional_decoder": false, // use https://arxiv.org/abs/1907.09006. Use it, if attention does not work well with your dataset. + "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ + "ddc_r": 7, // reduction rate for coarse decoder. // STOPNET "stopnet": true, // Train stopnet predicting the end of synthesis. From b4ac68df7ba1f50c3f8f40cd61962a773367a661 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:27:40 +0200 Subject: [PATCH 28/68] add DDC loss --- layers/losses.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/layers/losses.py b/layers/losses.py index 608e247d..f7745b6e 100644 --- a/layers/losses.py +++ b/layers/losses.py @@ -184,7 +184,7 @@ class TacotronLoss(torch.nn.Module): def forward(self, postnet_output, decoder_output, mel_input, linear_input, stopnet_output, stopnet_target, output_lens, decoder_b_output, - alignments, alignment_lens, input_lens): + alignments, alignment_lens, alignments_backwards, input_lens): return_dict = {} # decoder and postnet losses @@ -226,6 +226,15 @@ class TacotronLoss(torch.nn.Module): return_dict['decoder_b_loss'] = decoder_b_loss return_dict['decoder_c_loss'] = decoder_c_loss + # double decoder consistency loss (if enabled) + if self.config.double_decoder_consistency: + decoder_b_loss = self.criterion(decoder_b_output, mel_input, output_lens) + # decoder_c_loss = torch.nn.functional.l1_loss(decoder_b_output, decoder_output) + attention_c_loss = torch.nn.functional.l1_loss(alignments, alignments_backwards) + loss += decoder_b_loss + attention_c_loss + return_dict['decoder_coarse_loss'] = decoder_b_loss + return_dict['decoder_ddc_loss'] = attention_c_loss + # guided attention loss (if enabled) if self.config.ga_alpha > 0: ga_loss = self.criterion_ga(alignments, input_lens, alignment_lens) From bd7237d916823ffd9d3832360aa057b5203947b2 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:28:16 +0200 Subject: [PATCH 29/68] inherit TacotronAbstact with both tacotron and tacotron2 --- models/tacotron.py | 135 ++++++++++++++++++------------------------- models/tacotron2.py | 136 ++++++++++++++++++++++++++++---------------- 2 files changed, 142 insertions(+), 129 deletions(-) diff --git a/models/tacotron.py b/models/tacotron.py index fba82b1b..c526374a 100644 --- a/models/tacotron.py +++ b/models/tacotron.py @@ -1,23 +1,21 @@ # coding: utf-8 import torch -import copy from torch import nn -from TTS.layers.tacotron import Encoder, Decoder, PostCBHG -from TTS.utils.generic_utils import sequence_mask + from TTS.layers.gst_layers import GST +from TTS.layers.tacotron import Decoder, Encoder, PostCBHG +from TTS.models.tacotron_abstract import TacotronAbstract -class Tacotron(nn.Module): +class Tacotron(TacotronAbstract): def __init__(self, num_chars, num_speakers, r=5, postnet_output_dim=1025, decoder_output_dim=80, - memory_size=5, attn_type='original', attn_win=False, - gst=False, attn_norm="sigmoid", prenet_type="original", prenet_dropout=True, @@ -27,38 +25,41 @@ class Tacotron(nn.Module): location_attn=True, attn_K=5, separate_stopnet=True, - bidirectional_decoder=False): - super(Tacotron, self).__init__() - self.r = r - self.decoder_output_dim = decoder_output_dim - self.postnet_output_dim = postnet_output_dim - self.gst = gst - self.num_speakers = num_speakers - self.bidirectional_decoder = bidirectional_decoder - decoder_dim = 512 if num_speakers > 1 else 256 - encoder_dim = 512 if num_speakers > 1 else 256 + bidirectional_decoder=False, + double_decoder_consistency=False, + ddc_r=None, + gst=False, + memory_size=5): + super(Tacotron, + self).__init__(num_chars, num_speakers, r, postnet_output_dim, + decoder_output_dim, attn_type, attn_win, + attn_norm, prenet_type, prenet_dropout, + forward_attn, trans_agent, forward_attn_mask, + location_attn, attn_K, separate_stopnet, + bidirectional_decoder, double_decoder_consistency, + ddc_r, gst) + decoder_in_features = 512 if num_speakers > 1 else 256 + encoder_in_features = 512 if num_speakers > 1 else 256 + speaker_embedding_dim = 256 proj_speaker_dim = 80 if num_speakers > 1 else 0 - # embedding layer + # base model layers self.embedding = nn.Embedding(num_chars, 256, padding_idx=0) self.embedding.weight.data.normal_(0, 0.3) - # boilerplate model - self.encoder = Encoder(encoder_dim) - self.decoder = Decoder(decoder_dim, decoder_output_dim, r, memory_size, attn_type, attn_win, + self.encoder = Encoder(encoder_in_features) + self.decoder = Decoder(decoder_in_features, decoder_output_dim, r, memory_size, attn_type, attn_win, attn_norm, prenet_type, prenet_dropout, forward_attn, trans_agent, forward_attn_mask, location_attn, attn_K, separate_stopnet, proj_speaker_dim) - if self.bidirectional_decoder: - self.decoder_backward = copy.deepcopy(self.decoder) self.postnet = PostCBHG(decoder_output_dim) self.last_linear = nn.Linear(self.postnet.cbhg.gru_features * 2, postnet_output_dim) # speaker embedding layers if num_speakers > 1: - self.speaker_embedding = nn.Embedding(num_speakers, 256) + self.speaker_embedding = nn.Embedding(num_speakers, speaker_embedding_dim) self.speaker_embedding.weight.data.normal_(0, 0.3) self.speaker_project_mel = nn.Sequential( - nn.Linear(256, proj_speaker_dim), nn.Tanh()) + nn.Linear(speaker_embedding_dim, proj_speaker_dim), nn.Tanh()) self.speaker_embeddings = None self.speaker_embeddings_projected = None # global style token layers @@ -68,28 +69,15 @@ class Tacotron(nn.Module): num_heads=4, num_style_tokens=10, embedding_dim=gst_embedding_dim) + # backward pass decoder + if self.bidirectional_decoder: + self._init_backward_decoder() + # setup DDC + if self.double_decoder_consistency: + self._init_coarse_decoder() - def _init_states(self): - self.speaker_embeddings = None - self.speaker_embeddings_projected = None - def compute_speaker_embedding(self, speaker_ids): - if hasattr(self, "speaker_embedding") and speaker_ids is None: - raise RuntimeError( - " [!] Model has speaker embedding layer but speaker_id is not provided" - ) - if hasattr(self, "speaker_embedding") and speaker_ids is not None: - self.speaker_embeddings = self._compute_speaker_embedding( - speaker_ids) - self.speaker_embeddings_projected = self.speaker_project_mel( - self.speaker_embeddings).squeeze(1) - - def compute_gst(self, inputs, mel_specs): - gst_outputs = self.gst_layer(mel_specs) - inputs = self._add_speaker_embedding(inputs, gst_outputs) - return inputs - - def forward(self, characters, text_lengths, mel_specs, speaker_ids=None): + def forward(self, characters, text_lengths, mel_specs, mel_lengths=None, speaker_ids=None): """ Shapes: - characters: B x T_in @@ -98,45 +86,59 @@ class Tacotron(nn.Module): - speaker_ids: B x 1 """ self._init_states() - mask = sequence_mask(text_lengths).to(characters.device) + input_mask, output_mask = self.compute_masks(text_lengths, mel_lengths) # B x T_in x embed_dim inputs = self.embedding(characters) # B x speaker_embed_dim - self.compute_speaker_embedding(speaker_ids) + if speaker_ids is not None: + self.compute_speaker_embedding(speaker_ids) if self.num_speakers > 1: # B x T_in x embed_dim + speaker_embed_dim inputs = self._concat_speaker_embedding(inputs, self.speaker_embeddings) - # B x T_in x encoder_dim + # B x T_in x encoder_in_features encoder_outputs = self.encoder(inputs) + # sequence masking + encoder_outputs = encoder_outputs * input_mask.unsqueeze(2).expand_as(encoder_outputs) + # global style token if self.gst: # B x gst_dim encoder_outputs = self.compute_gst(encoder_outputs, mel_specs) if self.num_speakers > 1: encoder_outputs = self._concat_speaker_embedding( encoder_outputs, self.speaker_embeddings) - # decoder_outputs: B x decoder_dim x T_out - # alignments: B x T_in x encoder_dim + # decoder_outputs: B x decoder_in_features x T_out + # alignments: B x T_in x encoder_in_features # stop_tokens: B x T_in decoder_outputs, alignments, stop_tokens = self.decoder( - encoder_outputs, mel_specs, mask, + encoder_outputs, mel_specs, input_mask, self.speaker_embeddings_projected) - # B x T_out x decoder_dim + # sequence masking + if output_mask is not None: + decoder_outputs = decoder_outputs * output_mask.unsqueeze(1).expand_as(decoder_outputs) + # B x T_out x decoder_in_features postnet_outputs = self.postnet(decoder_outputs) + # sequence masking + if output_mask is not None: + postnet_outputs = postnet_outputs * output_mask.unsqueeze(2).expand_as(postnet_outputs) # B x T_out x posnet_dim postnet_outputs = self.last_linear(postnet_outputs) - # B x T_out x decoder_dim + # B x T_out x decoder_in_features decoder_outputs = decoder_outputs.transpose(1, 2).contiguous() if self.bidirectional_decoder: - decoder_outputs_backward, alignments_backward = self._backward_inference(mel_specs, encoder_outputs, mask) + decoder_outputs_backward, alignments_backward = self._backward_pass(mel_specs, encoder_outputs, input_mask) return decoder_outputs, postnet_outputs, alignments, stop_tokens, decoder_outputs_backward, alignments_backward + if self.double_decoder_consistency: + decoder_outputs_backward, alignments_backward = self._coarse_decoder_pass(mel_specs, encoder_outputs, alignments, input_mask) + return decoder_outputs, postnet_outputs, alignments, stop_tokens, decoder_outputs_backward, alignments_backward return decoder_outputs, postnet_outputs, alignments, stop_tokens @torch.no_grad() def inference(self, characters, speaker_ids=None, style_mel=None): inputs = self.embedding(characters) self._init_states() - self.compute_speaker_embedding(speaker_ids) + if speaker_ids is not None: + self.compute_speaker_embedding(speaker_ids) if self.num_speakers > 1: inputs = self._concat_speaker_embedding(inputs, self.speaker_embeddings) @@ -152,28 +154,3 @@ class Tacotron(nn.Module): postnet_outputs = self.last_linear(postnet_outputs) decoder_outputs = decoder_outputs.transpose(1, 2) return decoder_outputs, postnet_outputs, alignments, stop_tokens - - def _backward_inference(self, mel_specs, encoder_outputs, mask): - decoder_outputs_b, alignments_b, _ = self.decoder_backward( - encoder_outputs, torch.flip(mel_specs, dims=(1,)), mask, - self.speaker_embeddings_projected) - decoder_outputs_b = decoder_outputs_b.transpose(1, 2).contiguous() - return decoder_outputs_b, alignments_b - - def _compute_speaker_embedding(self, speaker_ids): - speaker_embeddings = self.speaker_embedding(speaker_ids) - return speaker_embeddings.unsqueeze_(1) - - @staticmethod - def _add_speaker_embedding(outputs, speaker_embeddings): - speaker_embeddings_ = speaker_embeddings.expand( - outputs.size(0), outputs.size(1), -1) - outputs = outputs + speaker_embeddings_ - return outputs - - @staticmethod - def _concat_speaker_embedding(outputs, speaker_embeddings): - speaker_embeddings_ = speaker_embeddings.expand( - outputs.size(0), outputs.size(1), -1) - outputs = torch.cat([outputs, speaker_embeddings_], dim=-1) - return outputs diff --git a/models/tacotron2.py b/models/tacotron2.py index 3e7adfca..bbce4be9 100644 --- a/models/tacotron2.py +++ b/models/tacotron2.py @@ -1,13 +1,15 @@ -import copy -import torch from math import sqrt + +import torch from torch import nn -from TTS.layers.tacotron2 import Encoder, Decoder, Postnet -from TTS.utils.generic_utils import sequence_mask + +from TTS.layers.gst_layers import GST +from TTS.layers.tacotron2 import Decoder, Encoder, Postnet +from TTS.models.tacotron_abstract import TacotronAbstract # TODO: match function arguments with tacotron -class Tacotron2(nn.Module): +class Tacotron2(TacotronAbstract): def __init__(self, num_chars, num_speakers, @@ -25,16 +27,22 @@ class Tacotron2(nn.Module): location_attn=True, attn_K=5, separate_stopnet=True, - bidirectional_decoder=False): - super(Tacotron2, self).__init__() - self.postnet_output_dim = postnet_output_dim - self.decoder_output_dim = decoder_output_dim - self.r = r - self.bidirectional_decoder = bidirectional_decoder - decoder_dim = 512 if num_speakers > 1 else 512 - encoder_dim = 512 if num_speakers > 1 else 512 + bidirectional_decoder=False, + double_decoder_consistency=False, + ddc_r=None, + gst=False): + super(Tacotron2, + self).__init__(num_chars, num_speakers, r, postnet_output_dim, + decoder_output_dim, attn_type, attn_win, + attn_norm, prenet_type, prenet_dropout, + forward_attn, trans_agent, forward_attn_mask, + location_attn, attn_K, separate_stopnet, + bidirectional_decoder, double_decoder_consistency, + ddc_r, gst) + decoder_in_features = 512 if num_speakers > 1 else 512 + encoder_in_features = 512 if num_speakers > 1 else 512 proj_speaker_dim = 80 if num_speakers > 1 else 0 - # embedding layer + # base layers self.embedding = nn.Embedding(num_chars, 512, padding_idx=0) std = sqrt(2.0 / (num_chars + 512)) val = sqrt(3.0) * std # uniform bounds for std @@ -42,20 +50,25 @@ class Tacotron2(nn.Module): if num_speakers > 1: self.speaker_embedding = nn.Embedding(num_speakers, 512) self.speaker_embedding.weight.data.normal_(0, 0.3) - self.speaker_embeddings = None - self.speaker_embeddings_projected = None - self.encoder = Encoder(encoder_dim) - self.decoder = Decoder(decoder_dim, self.decoder_output_dim, r, attn_type, attn_win, + self.encoder = Encoder(encoder_in_features) + self.decoder = Decoder(decoder_in_features, self.decoder_output_dim, r, attn_type, attn_win, attn_norm, prenet_type, prenet_dropout, forward_attn, trans_agent, forward_attn_mask, location_attn, attn_K, separate_stopnet, proj_speaker_dim) - if self.bidirectional_decoder: - self.decoder_backward = copy.deepcopy(self.decoder) self.postnet = Postnet(self.postnet_output_dim) - - def _init_states(self): - self.speaker_embeddings = None - self.speaker_embeddings_projected = None + # global style token layers + if self.gst: + gst_embedding_dim = encoder_in_features + self.gst_layer = GST(num_mel=80, + num_heads=4, + num_style_tokens=10, + embedding_dim=gst_embedding_dim) + # backward pass decoder + if self.bidirectional_decoder: + self._init_backward_decoder() + # setup DDC + if self.double_decoder_consistency: + self._init_coarse_decoder() @staticmethod def shape_outputs(mel_outputs, mel_outputs_postnet, alignments): @@ -63,31 +76,60 @@ class Tacotron2(nn.Module): mel_outputs_postnet = mel_outputs_postnet.transpose(1, 2) return mel_outputs, mel_outputs_postnet, alignments - def forward(self, text, text_lengths, mel_specs=None, speaker_ids=None): + def forward(self, text, text_lengths, mel_specs=None, mel_lengths=None, speaker_ids=None): self._init_states() # compute mask for padding - mask = sequence_mask(text_lengths).to(text.device) + # B x T_in_max (boolean) + input_mask, output_mask = self.compute_masks(text_lengths, mel_lengths) + # B x D_embed x T_in_max embedded_inputs = self.embedding(text).transpose(1, 2) + # B x T_in_max x D_en encoder_outputs = self.encoder(embedded_inputs, text_lengths) - encoder_outputs = self._add_speaker_embedding(encoder_outputs, - speaker_ids) + # adding speaker embeddding to encoder output + # TODO: multi-speaker + # B x speaker_embed_dim + if speaker_ids is not None: + self.compute_speaker_embedding(speaker_ids) + if self.num_speakers > 1: + # B x T_in x embed_dim + speaker_embed_dim + encoder_outputs = self._add_speaker_embedding(encoder_outputs, + self.speaker_embeddings) + encoder_outputs = encoder_outputs * input_mask.unsqueeze(2).expand_as(encoder_outputs) + # global style token + if self.gst: + # B x gst_dim + encoder_outputs = self.compute_gst(encoder_outputs, mel_specs) + # B x mel_dim x T_out -- B x T_out//r x T_in -- B x T_out//r decoder_outputs, alignments, stop_tokens = self.decoder( - encoder_outputs, mel_specs, mask) + encoder_outputs, mel_specs, input_mask) + # sequence masking + if mel_lengths is not None: + decoder_outputs = decoder_outputs * output_mask.unsqueeze(1).expand_as(decoder_outputs) + # B x mel_dim x T_out postnet_outputs = self.postnet(decoder_outputs) - postnet_outputs = decoder_outputs + postnet_outputs + # sequence masking + if output_mask is not None: + postnet_outputs = postnet_outputs * output_mask.unsqueeze(1).expand_as(postnet_outputs) + # B x T_out x mel_dim -- B x T_out x mel_dim -- B x T_out//r x T_in decoder_outputs, postnet_outputs, alignments = self.shape_outputs( decoder_outputs, postnet_outputs, alignments) if self.bidirectional_decoder: - decoder_outputs_backward, alignments_backward = self._backward_inference(mel_specs, encoder_outputs, mask) + decoder_outputs_backward, alignments_backward = self._backward_pass(mel_specs, encoder_outputs, input_mask) return decoder_outputs, postnet_outputs, alignments, stop_tokens, decoder_outputs_backward, alignments_backward + if self.double_decoder_consistency: + decoder_outputs_backward, alignments_backward = self._coarse_decoder_pass(mel_specs, encoder_outputs, alignments, input_mask) + return decoder_outputs, postnet_outputs, alignments, stop_tokens, decoder_outputs_backward, alignments_backward return decoder_outputs, postnet_outputs, alignments, stop_tokens @torch.no_grad() def inference(self, text, speaker_ids=None): embedded_inputs = self.embedding(text).transpose(1, 2) encoder_outputs = self.encoder.inference(embedded_inputs) - encoder_outputs = self._add_speaker_embedding(encoder_outputs, - speaker_ids) + if speaker_ids is not None: + self.compute_speaker_embedding(speaker_ids) + if self.num_speakers > 1: + encoder_outputs = self._add_speaker_embedding(encoder_outputs, + self.speaker_embeddings) mel_outputs, alignments, stop_tokens = self.decoder.inference( encoder_outputs) mel_outputs_postnet = self.postnet(mel_outputs) @@ -112,22 +154,16 @@ class Tacotron2(nn.Module): mel_outputs, mel_outputs_postnet, alignments) return mel_outputs, mel_outputs_postnet, alignments, stop_tokens - def _backward_inference(self, mel_specs, encoder_outputs, mask): - decoder_outputs_b, alignments_b, _ = self.decoder_backward( - encoder_outputs, torch.flip(mel_specs, dims=(1,)), mask, - self.speaker_embeddings_projected) - decoder_outputs_b = decoder_outputs_b.transpose(1, 2) - return decoder_outputs_b, alignments_b - def _add_speaker_embedding(self, encoder_outputs, speaker_ids): - if hasattr(self, "speaker_embedding") and speaker_ids is None: - raise RuntimeError(" [!] Model has speaker embedding layer but speaker_id is not provided") - if hasattr(self, "speaker_embedding") and speaker_ids is not None: - speaker_embeddings = self.speaker_embedding(speaker_ids) + def _speaker_embedding_pass(self, encoder_outputs, speaker_ids): + # TODO: multi-speaker + # if hasattr(self, "speaker_embedding") and speaker_ids is None: + # raise RuntimeError(" [!] Model has speaker embedding layer but speaker_id is not provided") + # if hasattr(self, "speaker_embedding") and speaker_ids is not None: - speaker_embeddings.unsqueeze_(1) - speaker_embeddings = speaker_embeddings.expand(encoder_outputs.size(0), - encoder_outputs.size(1), - -1) - encoder_outputs = encoder_outputs + speaker_embeddings - return encoder_outputs + # speaker_embeddings = speaker_embeddings.expand(encoder_outputs.size(0), + # encoder_outputs.size(1), + # -1) + # encoder_outputs = encoder_outputs + speaker_embeddings + # return encoder_outputs + pass From e272e07ce99f6789dbf5461e3daec1e3b4bc3fc3 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:28:55 +0200 Subject: [PATCH 30/68] update dummy model config --- tests/outputs/dummy_model_config.json | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/outputs/dummy_model_config.json b/tests/outputs/dummy_model_config.json index 2f56c6ce..d301b61a 100644 --- a/tests/outputs/dummy_model_config.json +++ b/tests/outputs/dummy_model_config.json @@ -4,7 +4,7 @@ "audio":{ // Audio processing parameters - "num_mels": 80, // size of the mel spec frame. + "num_mels": 80, // size of the mel spec frame. "num_freq": 1025, // number of stft frequency levels. Size of the linear spectogram frame. "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. "frame_length_ms": 50, // stft window length in ms. @@ -31,19 +31,19 @@ "reinit_layers": [], - "model": "Tacotron2", // one of the model in models/ + "model": "Tacotron2", // one of the model in models/ "grad_clip": 1, // upper limit for gradients for clipping. "epochs": 1000, // total number of epochs to train. "lr": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. "lr_decay": false, // if true, Noam learning rate decaying is applied through training. "warmup_steps": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" "windowing": false, // Enables attention windowing. Used only in eval mode. - "memory_size": 5, // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. + "memory_size": 5, // ONLY TACOTRON - memory queue size used to queue network predictions to feed autoregressive connection. Useful if r < 5. "attention_norm": "sigmoid", // softmax or sigmoid. Suggested to use softmax for Tacotron2 and sigmoid for Tacotron. "prenet_type": "original", // ONLY TACOTRON2 - "original" or "bn". - "prenet_dropout": true, // ONLY TACOTRON2 - enable/disable dropout at prenet. + "prenet_dropout": true, // ONLY TACOTRON2 - enable/disable dropout at prenet. "use_forward_attn": true, // ONLY TACOTRON2 - if it uses forward attention. In general, it aligns faster. - "forward_attn_mask": false, + "forward_attn_mask": false, "attention_type": "original", "attention_heads": 5, "bidirectional_decoder": false, @@ -51,13 +51,15 @@ "location_attn": false, // ONLY TACOTRON2 - enable_disable location sensitive attention. It is enabled for TACOTRON by default. "loss_masking": true, // enable / disable loss masking against the sequence padding. "enable_eos_bos_chars": false, // enable/disable beginning of sentence and end of sentence chars. - "stopnet": true, // Train stopnet predicting the end of synthesis. + "stopnet": true, // Train stopnet predicting the end of synthesis. "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - "use_gst": false, - + "use_gst": false, + "double_decoder_consistency": true, // use DDC explained here https://erogol.com/solving-attention-problems-of-tts-models-with-double-decoder-consistency-draft/ + "ddc_r": 7, // reduction rate for coarse decoder. + "batch_size": 32, // Batch size for training. Lower values than 32 might cause hard to learn attention. - "eval_batch_size":16, + "eval_batch_size":16, "r": 1, // Number of frames to predict for step. "wd": 0.000001, // Weight decay weight. "checkpoint": true, // If true, it saves checkpoints per "save_step" From 43d278be2f744cef27bfb25e100f74f0e6952a40 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:29:37 +0200 Subject: [PATCH 31/68] update model tests for ddc --- tests/test_tacotron2_model.py | 2 +- tests/test_tacotron_model.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_tacotron2_model.py b/tests/test_tacotron2_model.py index eb91b3cc..cf1d0778 100644 --- a/tests/test_tacotron2_model.py +++ b/tests/test_tacotron2_model.py @@ -51,7 +51,7 @@ class TacotronTrainTest(unittest.TestCase): optimizer = optim.Adam(model.parameters(), lr=c.lr) for i in range(5): mel_out, mel_postnet_out, align, stop_tokens = model.forward( - input, input_lengths, mel_spec, speaker_ids) + input, input_lengths, mel_spec, mel_lengths, speaker_ids) assert torch.sigmoid(stop_tokens).data.max() <= 1.0 assert torch.sigmoid(stop_tokens).data.min() >= 0.0 optimizer.zero_grad() diff --git a/tests/test_tacotron_model.py b/tests/test_tacotron_model.py index 7053a580..2bbb3c8d 100644 --- a/tests/test_tacotron_model.py +++ b/tests/test_tacotron_model.py @@ -66,7 +66,7 @@ class TacotronTrainTest(unittest.TestCase): optimizer = optim.Adam(model.parameters(), lr=c.lr) for _ in range(5): mel_out, linear_out, align, stop_tokens = model.forward( - input_dummy, input_lengths, mel_spec, speaker_ids) + input_dummy, input_lengths, mel_spec, mel_lengths, speaker_ids) optimizer.zero_grad() loss = criterion(mel_out, mel_spec, mel_lengths) stop_loss = criterion_st(stop_tokens, stop_targets) @@ -95,6 +95,7 @@ class TacotronGSTTrainTest(unittest.TestCase): mel_spec = torch.rand(8, 120, c.audio['num_mels']).to(device) linear_spec = torch.rand(8, 120, c.audio['num_freq']).to(device) mel_lengths = torch.randint(20, 120, (8, )).long().to(device) + mel_lengths[-1] = 120 stop_targets = torch.zeros(8, 120, 1).float().to(device) speaker_ids = torch.randint(0, 5, (8, )).long().to(device) @@ -130,7 +131,7 @@ class TacotronGSTTrainTest(unittest.TestCase): optimizer = optim.Adam(model.parameters(), lr=c.lr) for _ in range(10): mel_out, linear_out, align, stop_tokens = model.forward( - input_dummy, input_lengths, mel_spec, speaker_ids) + input_dummy, input_lengths, mel_spec, mel_lengths, speaker_ids) optimizer.zero_grad() loss = criterion(mel_out, mel_spec, mel_lengths) stop_loss = criterion_st(stop_tokens, stop_targets) From 0f88f8ec5a90ec0cf6cc355e1f45a2a77930deb6 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:30:12 +0200 Subject: [PATCH 32/68] train.py update for DDC --- train.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/train.py b/train.py index 5a345b59..612ee8a6 100644 --- a/train.py +++ b/train.py @@ -158,13 +158,14 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, optimizer_st.zero_grad() # forward pass model - if c.bidirectional_decoder: + if c.bidirectional_decoder or c.double_decoder_consistency: decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( - text_input, text_lengths, mel_input, speaker_ids=speaker_ids) + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids) else: decoder_output, postnet_output, alignments, stop_tokens = model( - text_input, text_lengths, mel_input, speaker_ids=speaker_ids) + text_input, text_lengths, mel_input, mel_lengths, speaker_ids=speaker_ids) decoder_backward_output = None + alignments_backward = None # set the alignment lengths wrt reduction factor for guided attention if mel_lengths.max() % model.decoder.r != 0: @@ -176,7 +177,8 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, loss_dict = criterion(postnet_output, decoder_output, mel_input, linear_input, stop_tokens, stop_targets, mel_lengths, decoder_backward_output, - alignments, alignment_lengths, text_lengths) + alignments, alignment_lengths, alignments_backward, + text_lengths) if c.bidirectional_decoder: keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_backward_loss'].item(), 'avg_decoder_c_loss': loss_dict['decoder_c_loss'].item()}) From 66ec1b2e2a89800a2b7ca93ce0dd50373c2bd99c Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:30:43 +0200 Subject: [PATCH 33/68] setup_model update following TacotronAbstract --- utils/generic_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 5b135061..3fbe84b1 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -160,13 +160,16 @@ def setup_model(num_chars, num_speakers, c): location_attn=c.location_attn, attn_K=c.attention_heads, separate_stopnet=c.separate_stopnet, - bidirectional_decoder=c.bidirectional_decoder) + bidirectional_decoder=c.bidirectional_decoder, + double_decoder_consistency=c.double_decoder_consistency, + ddc_r=c.ddc_r) elif c.model.lower() == "tacotron2": model = MyModel(num_chars=num_chars, num_speakers=num_speakers, r=c.r, postnet_output_dim=c.audio['num_mels'], decoder_output_dim=c.audio['num_mels'], + gst=c.use_gst, attn_type=c.attention_type, attn_win=c.windowing, attn_norm=c.attention_norm, @@ -178,7 +181,9 @@ def setup_model(num_chars, num_speakers, c): location_attn=c.location_attn, attn_K=c.attention_heads, separate_stopnet=c.separate_stopnet, - bidirectional_decoder=c.bidirectional_decoder) + bidirectional_decoder=c.bidirectional_decoder, + double_decoder_consistency=c.double_decoder_consistency, + ddc_r=c.ddc_r) return model class KeepAverage(): From a82f7d129d7b5c3068d4c19dc86bf93ce09f330a Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:31:17 +0200 Subject: [PATCH 34/68] check DDC parameters in config.json --- utils/generic_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/generic_utils.py b/utils/generic_utils.py index 3fbe84b1..eb6d21a9 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -318,6 +318,8 @@ def check_config(c): _check_argument('transition_agent', c, restricted=True, val_type=bool) _check_argument('location_attn', c, restricted=True, val_type=bool) _check_argument('bidirectional_decoder', c, restricted=True, val_type=bool) + _check_argument('double_decoder_consistency', c, restricted=True, val_type=bool) + _check_argument('ddc_r', c, restricted='double_decoder_consistency' in c.keys(), min_val=1, max_val=7, val_type=int) # stopnet _check_argument('stopnet', c, restricted=True, val_type=bool) From fd8f1ecb7d776918980379b59d79cc94e0970644 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 4 Jun 2020 14:58:18 +0200 Subject: [PATCH 35/68] lint updates --- models/tacotron_abstract.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/models/tacotron_abstract.py b/models/tacotron_abstract.py index db6cbdee..75a1a5cd 100644 --- a/models/tacotron_abstract.py +++ b/models/tacotron_abstract.py @@ -4,12 +4,12 @@ from abc import ABC, abstractmethod import torch from torch import nn -from TTS.layers.gst_layers import GST from TTS.utils.generic_utils import sequence_mask class TacotronAbstract(ABC, nn.Module): - def __init__(self, num_chars, + def __init__(self, + num_chars, num_speakers, r, postnet_output_dim=80, @@ -31,6 +31,7 @@ class TacotronAbstract(ABC, nn.Module): gst=False): """ Abstract Tacotron class """ super().__init__() + self.num_chars = num_chars self.r = r self.decoder_output_dim = decoder_output_dim self.postnet_output_dim = postnet_output_dim @@ -39,6 +40,17 @@ class TacotronAbstract(ABC, nn.Module): self.bidirectional_decoder = bidirectional_decoder self.double_decoder_consistency = double_decoder_consistency self.ddc_r = ddc_r + self.attn_type = attn_type + self.attn_win = attn_win + self.attn_norm = attn_norm + self.prenet_type = prenet_type + self.prenet_dropout = prenet_dropout + self.forward_attn = forward_attn + self.trans_agent = trans_agent + self.forward_attn_mask = forward_attn_mask + self.location_attn = location_attn + self.attn_K = attn_K + self.separate_stopnet = separate_stopnet # layers self.embedding = None @@ -48,9 +60,16 @@ class TacotronAbstract(ABC, nn.Module): # global style token if self.gst: - gst_embedding_dim = None self.gst_layer = None + # model states + self.speaker_embeddings = None + self.speaker_embeddings_projected = None + + # additional layers + self.decoder_backward = None + self.coarse_decoder = None + ############################# # INIT FUNCTIONS ############################# @@ -114,7 +133,7 @@ class TacotronAbstract(ABC, nn.Module): (0, 0, 0, padding_size, 0, 0)) decoder_outputs_backward, alignments_backward, _ = self.coarse_decoder( encoder_outputs.detach(), mel_specs, input_mask) - scale_factor = self.decoder.r_init / self.decoder.r + # scale_factor = self.decoder.r_init / self.decoder.r alignments_backward = torch.nn.functional.interpolate( alignments_backward.transpose(1, 2), size=alignments.shape[1], @@ -141,6 +160,7 @@ class TacotronAbstract(ABC, nn.Module): def compute_gst(self, inputs, mel_specs): """ Compute global style token """ + # pylint: disable=not-callable gst_outputs = self.gst_layer(mel_specs) inputs = self._add_speaker_embedding(inputs, gst_outputs) return inputs From 189227c7412da63f3f4253e3296c4b95f005ff93 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 5 Jun 2020 13:23:11 +0200 Subject: [PATCH 36/68] bug fix vocoder training --- vocoder/layers/losses.py | 4 ++-- vocoder/train.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index 22e2fb54..a24b129c 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -199,8 +199,8 @@ class GeneratorLoss(nn.Module): self.stft_loss_weight = C.stft_loss_weight self.subband_stft_loss_weight = C.subband_stft_loss_weight - self.mse_gan_loss_weight = C.mse_gan_loss_weight - self.hinge_gan_loss_weight = C.hinge_gan_loss_weight + self.mse_gan_loss_weight = C.mse_G_loss_weight + self.hinge_gan_loss_weight = C.hinge_G_loss_weight self.feat_match_loss_weight = C.feat_match_loss_weight if C.use_stft_loss: diff --git a/vocoder/train.py b/vocoder/train.py index a75c6a12..d1e6befe 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -173,7 +173,10 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, loss_dict = dict() for key, value in loss_G_dict.items(): - loss_dict[key] = value.item() + if isinstance(value, int): + loss_dict[key] = value + else: + loss_dict[key] = value.item() ############################## # DISCRIMINATOR From c5c3f6094cfe2eadc16d2c970fbe95cb2b3bb917 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 5 Jun 2020 13:25:53 +0200 Subject: [PATCH 37/68] bug fix for rwd discriminator --- vocoder/train.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vocoder/train.py b/vocoder/train.py index d1e6befe..82f3bee6 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -327,8 +327,12 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): y_hat = model_G.pqmf_synthesis(y_hat) y_G_sub = model_G.pqmf_analysis(y_G) - D_out_fake = model_D(y_hat) + if len(signature(model_D.forward).parameters) == 2: + D_out_fake = model_D(y_hat, c_G) + else: + D_out_fake = model_D(y_hat) D_out_real = None + if c.use_feat_match_loss: with torch.no_grad(): D_out_real = model_D(y_G) From 72fe5a8217f1e28439aec902f72bf8eaff7a7521 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 5 Jun 2020 13:27:09 +0200 Subject: [PATCH 38/68] correct name --- vocoder/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vocoder/train.py b/vocoder/train.py index 82f3bee6..2c3a2e0a 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -363,7 +363,7 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): for key, value in loss_G_dict.items(): update_eval_values['avg_' + key] = value.item() update_eval_values['avg_loader_time'] = loader_time - update_eval_values['avgP_step_time'] = step_time + update_eval_values['avg_step_time'] = step_time keep_avg.update_values(update_eval_values) # print eval stats From 81af469be053e6109808f2bbd73105ebd410efc8 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 5 Jun 2020 13:28:16 +0200 Subject: [PATCH 39/68] model config update --- .../multiband-melgan_and_rwd_config.json | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/vocoder/configs/multiband-melgan_and_rwd_config.json b/vocoder/configs/multiband-melgan_and_rwd_config.json index 9aeb7933..c783877d 100644 --- a/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -49,21 +49,28 @@ // LOSS PARAMETERS "use_stft_loss": true, + "use_subband_stft_loss": true, "use_mse_gan_loss": true, "use_hinge_gan_loss": false, "use_feat_match_loss": false, // use only with melgan discriminators - "stft_loss_weight": 1, + "stft_loss_weight": 0.5, + "subband_stft_loss_weight": 0.5, "mse_gan_loss_weight": 2.5, - "hinge_gan_loss_weight": 1, - "feat_match_loss_weight": 10.0, + "hinge_gan_loss_weight": 2.5, + "feat_match_loss_weight": 25.0, "stft_loss_params": { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240] }, - "target_loss": "avg_G_loss", // loss value to pick the best model + "subband_stft_loss_params":{ + "n_ffts": [384, 683, 171], + "hop_lengths": [30, 60, 10], + "win_lengths": [150, 300, 60] + }, + "target_loss": "avg_G_loss", // DISCRIMINATOR // "discriminator_model": "melgan_multiscale_discriminator", @@ -85,7 +92,7 @@ // GENERATOR "generator_model": "multiband_melgan_generator", "generator_model_params": { - "upsample_factors":[2 ,2, 4, 4], + "upsample_factors":[8, 4, 2], "num_res_blocks": 4 }, @@ -111,7 +118,7 @@ "noam_schedule": false, // use noam warmup and lr schedule. "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" "warmup_steps_disc": 4000, - "epochs": 1000, // total number of epochs to train. + "epochs": 100000, // total number of epochs to train. "wd": 0.000001, // Weight decay weight. "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. "lr_disc": 0.0001, @@ -131,6 +138,6 @@ "eval_split_size": 10, // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" + "output_path": "/data/rw/home/Trainings/" } From 79db7d931b213ed9a75d64fe8c5c5e1b27bcd863 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 5 Jun 2020 14:47:25 +0200 Subject: [PATCH 40/68] bug fix for DDC model eval run --- train.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 612ee8a6..13bda5ef 100644 --- a/train.py +++ b/train.py @@ -331,13 +331,14 @@ def evaluate(model, criterion, ap, global_step, epoch): assert mel_input.shape[1] % model.decoder.r == 0 # forward pass model - if c.bidirectional_decoder: + if c.bidirectional_decoder or c.double_decoder_consistency: decoder_output, postnet_output, alignments, stop_tokens, decoder_backward_output, alignments_backward = model( text_input, text_lengths, mel_input, speaker_ids=speaker_ids) else: decoder_output, postnet_output, alignments, stop_tokens = model( text_input, text_lengths, mel_input, speaker_ids=speaker_ids) decoder_backward_output = None + alignments_backward = None # set the alignment lengths wrt reduction factor for guided attention if mel_lengths.max() % model.decoder.r != 0: @@ -349,7 +350,8 @@ def evaluate(model, criterion, ap, global_step, epoch): loss_dict = criterion(postnet_output, decoder_output, mel_input, linear_input, stop_tokens, stop_targets, mel_lengths, decoder_backward_output, - alignments, alignment_lengths, text_lengths) + alignments, alignment_lengths, alignments_backward, + text_lengths) if c.bidirectional_decoder: keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_b_loss'].item(), 'avg_decoder_c_loss': loss_dict['decoder_c_loss'].item()}) From b86af6538f7044a0785cde6196c4b42748a6c04e Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 6 Jun 2020 12:59:59 +0200 Subject: [PATCH 41/68] bug fix for TTS model restore --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index 13bda5ef..a47828e0 100644 --- a/train.py +++ b/train.py @@ -542,7 +542,7 @@ def main(args): # pylint: disable=redefined-outer-name except: print(" > Partial model initialization.") model_dict = model.state_dict() - model_dict = set_init_dict(model_dict, checkpoint, c) + model_dict = set_init_dict(model_dict, checkpoint['model'], c) model.load_state_dict(model_dict) del model_dict for group in optimizer.param_groups: From fedb2542be72c6f96d10c0b379968337412b24ee Mon Sep 17 00:00:00 2001 From: erogol Date: Sat, 6 Jun 2020 13:00:46 +0200 Subject: [PATCH 42/68] bug fix vocoder model restore --- vocoder/configs/multiband-melgan_and_rwd_config.json | 4 ++-- vocoder/train.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vocoder/configs/multiband-melgan_and_rwd_config.json b/vocoder/configs/multiband-melgan_and_rwd_config.json index c783877d..736f3459 100644 --- a/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -56,8 +56,8 @@ "stft_loss_weight": 0.5, "subband_stft_loss_weight": 0.5, - "mse_gan_loss_weight": 2.5, - "hinge_gan_loss_weight": 2.5, + "mse_G_loss_weight": 2.5, + "hinge_G_loss_weight": 2.5, "feat_match_loss_weight": 25.0, "stft_loss_params": { diff --git a/vocoder/train.py b/vocoder/train.py index 2c3a2e0a..05805c68 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -421,7 +421,7 @@ def main(args): # pylint: disable=redefined-outer-name optimizer_gen.load_state_dict(checkpoint['optimizer']) model_disc.load_state_dict(checkpoint['model_disc']) optimizer_disc.load_state_dict(checkpoint['optimizer_disc']) - except KeyError: + except RuntimeError: print(" > Partial model initialization.") model_dict = model_gen.state_dict() model_dict = set_init_dict(model_dict, checkpoint['model'], c) From 2404f96cba51bdb642a65296e49bda905ae70830 Mon Sep 17 00:00:00 2001 From: erogol Date: Mon, 8 Jun 2020 03:22:17 +0200 Subject: [PATCH 43/68] bug fix and var renaming --- models/tacotron2.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/models/tacotron2.py b/models/tacotron2.py index bbce4be9..bce21e9e 100644 --- a/models/tacotron2.py +++ b/models/tacotron2.py @@ -107,6 +107,7 @@ class Tacotron2(TacotronAbstract): decoder_outputs = decoder_outputs * output_mask.unsqueeze(1).expand_as(decoder_outputs) # B x mel_dim x T_out postnet_outputs = self.postnet(decoder_outputs) + postnet_outputs = decoder_outputs + postnet_outputs # sequence masking if output_mask is not None: postnet_outputs = postnet_outputs * output_mask.unsqueeze(1).expand_as(postnet_outputs) @@ -130,13 +131,13 @@ class Tacotron2(TacotronAbstract): if self.num_speakers > 1: encoder_outputs = self._add_speaker_embedding(encoder_outputs, self.speaker_embeddings) - mel_outputs, alignments, stop_tokens = self.decoder.inference( + decoder_outputs, alignments, stop_tokens = self.decoder.inference( encoder_outputs) - mel_outputs_postnet = self.postnet(mel_outputs) - mel_outputs_postnet = mel_outputs + mel_outputs_postnet - mel_outputs, mel_outputs_postnet, alignments = self.shape_outputs( - mel_outputs, mel_outputs_postnet, alignments) - return mel_outputs, mel_outputs_postnet, alignments, stop_tokens + postnet_outputs = self.postnet(decoder_outputs) + postnet_outputs = decoder_outputs + postnet_outputs + decoder_outputs, postnet_outputs, alignments = self.shape_outputs( + decoder_outputs, postnet_outputs, alignments) + return decoder_outputs, postnet_outputs, alignments, stop_tokens def inference_truncated(self, text, speaker_ids=None): """ From 8d307f21336352949ff56caa29f3e582e4e7a115 Mon Sep 17 00:00:00 2001 From: erogol Date: Mon, 8 Jun 2020 10:43:09 +0200 Subject: [PATCH 44/68] run D in eval --- vocoder/configs/melgan_config.json | 4 +-- vocoder/train.py | 47 +++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json index bdfdf88a..0b6b16db 100644 --- a/vocoder/configs/melgan_config.json +++ b/vocoder/configs/melgan_config.json @@ -56,8 +56,8 @@ "stft_loss_weight": 0.5, "subband_stft_loss_weight": 0.5, - "mse_gan_loss_weight": 2.5, - "hinge_gan_loss_weight": 2.5, + "mse_G_loss_weight": 2.5, + "hinge_G_loss_weight": 2.5, "feat_match_loss_weight": 25, "stft_loss_params": { diff --git a/vocoder/train.py b/vocoder/train.py index 05805c68..54b36e6b 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -181,7 +181,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, ############################## # DISCRIMINATOR ############################## - if global_step > c.steps_to_start_discriminator: + if global_step >= c.steps_to_start_discriminator: # discriminator pass with torch.no_grad(): y_hat = model_G(c_D) @@ -295,7 +295,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, @torch.no_grad() -def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): +def evaluate(model_G, criterion_G, model_D, criterion_D, ap, global_step, epoch): data_loader = setup_loader(ap, is_val=True, verbose=(epoch == 0)) model_G.eval() model_D.eval() @@ -355,13 +355,52 @@ def evaluate(model_G, criterion_G, model_D, ap, global_step, epoch): for key, value in loss_G_dict.items(): loss_dict[key] = value.item() + ############################## + # DISCRIMINATOR + ############################## + + if global_step >= c.steps_to_start_discriminator: + # discriminator pass + with torch.no_grad(): + y_hat = model_G(c_G) + + # PQMF formatting + if y_hat.shape[1] > 1: + y_hat = model_G.pqmf_synthesis(y_hat) + + # run D with or without cond. features + if len(signature(model_D.forward).parameters) == 2: + D_out_fake = model_D(y_hat.detach(), c_G) + D_out_real = model_D(y_G, c_G) + else: + D_out_fake = model_D(y_hat.detach()) + D_out_real = model_D(y_G) + + # format D outputs + if isinstance(D_out_fake, tuple): + scores_fake, feats_fake = D_out_fake + if D_out_real is None: + scores_real, feats_real = None, None + else: + scores_real, feats_real = D_out_real + else: + scores_fake = D_out_fake + scores_real = D_out_real + + # compute losses + loss_D_dict = criterion_D(scores_fake, scores_real) + + for key, value in loss_D_dict.items(): + loss_dict[key] = value.item() + + step_time = time.time() - start_time epoch_time += step_time # update avg stats update_eval_values = dict() - for key, value in loss_G_dict.items(): - update_eval_values['avg_' + key] = value.item() + for key, value in loss_dict.items(): + update_eval_values['avg_' + key] = value update_eval_values['avg_loader_time'] = loader_time update_eval_values['avg_step_time'] = step_time keep_avg.update_values(update_eval_values) From dbe381a18647d60f9ee548f11162ec57fc50e5f0 Mon Sep 17 00:00:00 2001 From: erogol Date: Mon, 8 Jun 2020 10:45:38 +0200 Subject: [PATCH 45/68] linter fix --- models/tacotron2.py | 2 +- train.py | 2 +- vocoder/train.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/models/tacotron2.py b/models/tacotron2.py index bce21e9e..46b915b5 100644 --- a/models/tacotron2.py +++ b/models/tacotron2.py @@ -130,7 +130,7 @@ class Tacotron2(TacotronAbstract): self.compute_speaker_embedding(speaker_ids) if self.num_speakers > 1: encoder_outputs = self._add_speaker_embedding(encoder_outputs, - self.speaker_embeddings) + self.speaker_embeddings) decoder_outputs, alignments, stop_tokens = self.decoder.inference( encoder_outputs) postnet_outputs = self.postnet(decoder_outputs) diff --git a/train.py b/train.py index a47828e0..a5f3d2e7 100644 --- a/train.py +++ b/train.py @@ -350,7 +350,7 @@ def evaluate(model, criterion, ap, global_step, epoch): loss_dict = criterion(postnet_output, decoder_output, mel_input, linear_input, stop_tokens, stop_targets, mel_lengths, decoder_backward_output, - alignments, alignment_lengths, alignments_backward, + alignments, alignment_lengths, alignments_backward, text_lengths) if c.bidirectional_decoder: keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_b_loss'].item(), diff --git a/vocoder/train.py b/vocoder/train.py index 54b36e6b..259c5eb7 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -519,7 +519,7 @@ def main(args): # pylint: disable=redefined-outer-name model_disc, criterion_disc, optimizer_disc, scheduler_gen, scheduler_disc, ap, global_step, epoch) - eval_avg_loss_dict = evaluate(model_gen, criterion_gen, model_disc, ap, + eval_avg_loss_dict = evaluate(model_gen, criterion_gen, model_disc, criterion_disc, ap, global_step, epoch) c_logger.print_epoch_end(epoch, eval_avg_loss_dict) target_loss = eval_avg_loss_dict[c.target_loss] From 625ea70c98c67cdfa87dcbc71478a002370ac41a Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:01:13 +0200 Subject: [PATCH 46/68] TTS config update for using DDC --- config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config.json b/config.json index 74021970..5e10c535 100644 --- a/config.json +++ b/config.json @@ -1,7 +1,7 @@ { "model": "Tacotron2", - "run_name": "ljspeech", - "run_description": "tacotron2", + "run_name": "ljspeech-ddc-bn", + "run_description": "tacotron2 with ddc and batch-normalization", // AUDIO PARAMETERS "audio":{ @@ -83,8 +83,8 @@ // TACOTRON PRENET "memory_size": -1, // ONLY TACOTRON - size of the memory queue used fro storing last decoder predictions for auto-regression. If < 0, memory queue is disabled and decoder only uses the last prediction frame. - "prenet_type": "original", // "original" or "bn". - "prenet_dropout": true, // enable/disable dropout at prenet. + "prenet_type": "bn", // "original" or "bn". + "prenet_dropout": false, // enable/disable dropout at prenet. // ATTENTION "attention_type": "original", // 'original' or 'graves' @@ -124,7 +124,7 @@ // PHONEMES "phoneme_cache_path": "/media/erogol/data_ssd2/mozilla_us_phonemes_3", // phoneme computation is slow, therefore, it caches results in the given folder. - "use_phonemes": false, // use phonemes instead of raw characters. It is suggested for better pronounciation. + "use_phonemes": true, // use phonemes instead of raw characters. It is suggested for better pronounciation. "phoneme_language": "en-us", // depending on your target language, pick one from https://github.com/bootphon/phonemizer#languages // MULTI-SPEAKER and GST From 4ea649f59b87dd1bebbf31fa9a7015d765c21e35 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:01:34 +0200 Subject: [PATCH 47/68] change alignment_score to alignment_error --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index a5f3d2e7..869557e6 100644 --- a/train.py +++ b/train.py @@ -291,7 +291,7 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, "loss_postnet": keep_avg['avg_postnet_loss'], "loss_decoder": keep_avg['avg_decoder_loss'], "stopnet_loss": keep_avg['avg_stopnet_loss'], - "alignment_score": keep_avg['avg_align_error'], + "alignment_error": keep_avg['avg_align_error'], "epoch_time": epoch_time } if c.ga_alpha > 0: @@ -413,7 +413,7 @@ def evaluate(model, criterion, ap, global_step, epoch): "loss_postnet": keep_avg['avg_postnet_loss'], "loss_decoder": keep_avg['avg_decoder_loss'], "stopnet_loss": keep_avg['avg_stopnet_loss'], - "alignment_score": keep_avg['avg_align_error'], + "alignment_error": keep_avg['avg_align_error'], } if c.bidirectional_decoder: From f3aa295834007a956f115f53deed3e6b5907c9af Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:02:05 +0200 Subject: [PATCH 48/68] correction for vocoder configs --- vocoder/configs/melgan_config.json | 145 ------------------ .../multiband-melgan_and_rwd_config.json | 2 +- 2 files changed, 1 insertion(+), 146 deletions(-) delete mode 100644 vocoder/configs/melgan_config.json diff --git a/vocoder/configs/melgan_config.json b/vocoder/configs/melgan_config.json deleted file mode 100644 index 0b6b16db..00000000 --- a/vocoder/configs/melgan_config.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "run_name": "melgan", - "run_description": "melgan initial run", - - // AUDIO PARAMETERS - "audio":{ - // stft parameters - "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. - "win_length": 1024, // stft window length in ms. - "hop_length": 256, // stft window hop-lengh in ms. - "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. - "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. - - // Audio processing parameters - "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. - "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. - "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. - - // Silence trimming - "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) - "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. - - // MelSpectrogram parameters - "num_mels": 80, // size of the mel spec frame. - "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! - "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! - - // Normalization parameters - "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. - "min_level_db": -100, // lower bound for normalization - "symmetric_norm": true, // move normalization to range [-1, 1] - "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] - "clip_norm": true, // clip normalized values into the range. - "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored - }, - - // DISTRIBUTED TRAINING - // "distributed":{ - // "backend": "nccl", - // "url": "tcp:\/\/localhost:54321" - // }, - - // MODEL PARAMETERS - "use_pqmf": true, - - // LOSS PARAMETERS - "use_stft_loss": true, - "use_subband_stft_loss": true, - "use_mse_gan_loss": true, - "use_hinge_gan_loss": false, - "use_feat_match_loss": false, // use only with melgan discriminators - - "stft_loss_weight": 0.5, - "subband_stft_loss_weight": 0.5, - "mse_G_loss_weight": 2.5, - "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25, - - "stft_loss_params": { - "n_ffts": [1024, 2048, 512], - "hop_lengths": [120, 240, 50], - "win_lengths": [600, 1200, 240] - }, - "subband_stft_loss_params":{ - "n_ffts": [384, 683, 171], - "hop_lengths": [30, 60, 10], - "win_lengths": [150, 300, 60] - }, - - "target_loss": "avg_G_loss", // loss value to pick the best model - - // DISCRIMINATOR - "discriminator_model": "melgan_multiscale_discriminator", - "discriminator_model_params":{ - "base_channels": 16, - "max_channels":1024, - "downsample_factors":[4, 4, 4, 4] - }, - "steps_to_start_discriminator": 100000, // steps required to start GAN trainining.1 - - // "discriminator_model": "random_window_discriminator", - // "discriminator_model_params":{ - // "uncond_disc_donwsample_factors": [8, 4], - // "cond_disc_downsample_factors": [[8, 4, 2, 2, 2], [8, 4, 2, 2], [8, 4, 2], [8, 4], [4, 2, 2]], - // "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], - // "window_sizes": [512, 1024, 2048, 4096, 8192] - // }, - - - // GENERATOR - "generator_model": "multiband_melgan_generator", - "generator_model_params": { - "upsample_factors":[2 ,2, 4, 4], - "num_res_blocks": 4 - }, - - // DATASET - "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", - "seq_len": 16384, - "pad_short": 2000, - "conv_pad": 0, - "use_noise_augment": true, - "use_cache": true, - - "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. - - // TRAINING - "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. - - // VALIDATION - "run_eval": true, - "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. - "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. - - // OPTIMIZER - "noam_schedule": false, // use noam warmup and lr schedule. - "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" - "warmup_steps_disc": 4000, - "epochs": 1000, // total number of epochs to train. - "wd": 0.000001, // Weight decay weight. - "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0001, - "gen_clip_grad": 10.0, - "disc_clip_grad": 10.0, - - // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. - "checkpoint": true, // If true, it saves checkpoints per "save_step" - "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. - - // DATA LOADING - "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. - "num_val_loader_workers": 4, // number of evaluation data loader processes. - "eval_split_size": 10, - - // PATHS - "output_path": "/home/erogol/Models/LJSpeech/" -} - diff --git a/vocoder/configs/multiband-melgan_and_rwd_config.json b/vocoder/configs/multiband-melgan_and_rwd_config.json index 736f3459..f4b91aae 100644 --- a/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -79,7 +79,7 @@ // "max_channels":1024, // "downsample_factors":[4, 4, 4, 4] // }, - "steps_to_start_discriminator": 100000, // steps required to start GAN trainining.1 + "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 "discriminator_model": "random_window_discriminator", "discriminator_model_params":{ From aedbd318d55681d23562d4a84e843aea9b0bd396 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:02:28 +0200 Subject: [PATCH 49/68] remove weight norm for melgan generator --- vocoder/models/melgan_generator.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vocoder/models/melgan_generator.py b/vocoder/models/melgan_generator.py index 1e47816d..e69e6ef3 100644 --- a/vocoder/models/melgan_generator.py +++ b/vocoder/models/melgan_generator.py @@ -87,3 +87,12 @@ class MelganGenerator(nn.Module): (self.inference_padding, self.inference_padding), 'replicate') return self.layers(cond_features) + + def remove_weight_norm(self): + for _, layer in enumerate(self.layers): + if len(layer.state_dict()) != 0: + try: + nn.utils.remove_weight_norm(layer) + except ValueError: + layer.remove_weight_norm() + From 8552c1d991955c8545e59b7d1af492449ee82c9a Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:02:55 +0200 Subject: [PATCH 50/68] fix multiband melgan inference --- vocoder/models/multiband_melgan_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vocoder/models/multiband_melgan_generator.py b/vocoder/models/multiband_melgan_generator.py index 8feacd25..15e7426e 100644 --- a/vocoder/models/multiband_melgan_generator.py +++ b/vocoder/models/multiband_melgan_generator.py @@ -29,10 +29,11 @@ class MultibandMelganGenerator(MelganGenerator): def pqmf_synthesis(self, x): return self.pqmf_layer.synthesis(x) + @torch.no_grad() def inference(self, cond_features): cond_features = cond_features.to(self.layers[1].weight.device) cond_features = torch.nn.functional.pad( cond_features, (self.inference_padding, self.inference_padding), 'replicate') - return self.pqmf.synthesis(self.layers(cond_features)) + return self.pqmf_synthesis(self.layers(cond_features)) From f0144bfcba8411fd68a934158307ec49b00a0ea2 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:03:37 +0200 Subject: [PATCH 51/68] add lr schedulers for generator and discriminator --- vocoder/train.py | 56 +++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/vocoder/train.py b/vocoder/train.py index 259c5eb7..c563dff0 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -107,10 +107,6 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, global_step += 1 - # get current learning rates - current_lr_G = list(optimizer_G.param_groups)[0]['lr'] - current_lr_D = list(optimizer_D.param_groups)[0]['lr'] - ############################## # GENERATOR ############################## @@ -166,9 +162,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, torch.nn.utils.clip_grad_norm_(model_G.parameters(), c.gen_clip_grad) optimizer_G.step() - - # setup lr - if c.noam_schedule: + if scheduler_G is not None: scheduler_G.step() loss_dict = dict() @@ -221,9 +215,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, torch.nn.utils.clip_grad_norm_(model_D.parameters(), c.disc_clip_grad) optimizer_D.step() - - # setup lr - if c.noam_schedule: + if c.scheduler_D is not None: scheduler_D.step() for key, value in loss_D_dict.items(): @@ -232,6 +224,10 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, step_time = time.time() - start_time epoch_time += step_time + # get current learning rates + current_lr_G = list(optimizer_G.param_groups)[0]['lr'] + current_lr_D = list(optimizer_D.param_groups)[0]['lr'] + # update avg stats update_train_values = dict() for key, value in loss_dict.items(): @@ -244,7 +240,8 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, if global_step % c.print_step == 0: c_logger.print_train_step(batch_n_iter, num_iter, global_step, step_time, loader_time, current_lr_G, - loss_dict, keep_avg.avg_values) + current_lr_D, loss_dict, + keep_avg.avg_values) # plot step stats if global_step % 10 == 0: @@ -262,8 +259,10 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, # save model save_checkpoint(model_G, optimizer_G, + scheduler_G, model_D, optimizer_D, + scheduler_D, global_step, epoch, OUT_PATH, @@ -434,6 +433,7 @@ def main(args): # pylint: disable=redefined-outer-name # setup audio processor ap = AudioProcessor(**c.audio) + # DISTRUBUTED # if num_gpus > 1: # init_distributed(args.rank, num_gpus, args.group_id, @@ -449,6 +449,12 @@ def main(args): # pylint: disable=redefined-outer-name lr=c.lr_disc, weight_decay=0) + # schedulers + scheduler_gen = getattr(torch.optim.lr_scheduler, c.lr_scheduler_gen) + scheduler_disc = getattr(torch.optim.lr_scheduler, c.lr_scheduler_disc) + scheduler_gen = scheduler_gen(optimizer_gen, **c.lr_scheduler_gen_params) + scheduler_disc = scheduler_disc(optimizer_disc, **c.lr_scheduler_disc_params) + # setup criterion criterion_gen = GeneratorLoss(c) criterion_disc = DiscriminatorLoss(c) @@ -456,12 +462,26 @@ def main(args): # pylint: disable=redefined-outer-name if args.restore_path: checkpoint = torch.load(args.restore_path, map_location='cpu') try: + print(" > Restoring Generator Model...") model_gen.load_state_dict(checkpoint['model']) + print(" > Restoring Generator Optimizer...") optimizer_gen.load_state_dict(checkpoint['optimizer']) + print(" > Restoring Discriminator Model...") model_disc.load_state_dict(checkpoint['model_disc']) + print(" > Restoring Discriminator Optimizer...") optimizer_disc.load_state_dict(checkpoint['optimizer_disc']) + if 'scheduler' in checkpoint: + print(" > Restoring Generator LR Scheduler...") + scheduler_gen.load_state_dict(checkpoint['scheduler']) + # NOTE: Not sure if necessary + scheduler_gen.optimizer = optimizer_gen + if 'scheduler_disc' in checkpoint: + print(" > Restoring Discriminator LR Scheduler...") + scheduler_disc.load_state_dict(checkpoint['scheduler_disc']) + scheduler_disc.optimizer = optimizer_disc except RuntimeError: - print(" > Partial model initialization.") + # retore only matching layers. + print(" > Partial model initialization...") model_dict = model_gen.state_dict() model_dict = set_init_dict(model_dict, checkpoint['model'], c) model_gen.load_state_dict(model_dict) @@ -494,16 +514,6 @@ def main(args): # pylint: disable=redefined-outer-name # if num_gpus > 1: # model = apply_gradient_allreduce(model) - if c.noam_schedule: - scheduler_gen = NoamLR(optimizer_gen, - warmup_steps=c.warmup_steps_gen, - last_epoch=args.restore_step - 1) - scheduler_disc = NoamLR(optimizer_disc, - warmup_steps=c.warmup_steps_gen, - last_epoch=args.restore_step - 1) - else: - scheduler_gen, scheduler_disc = None, None - num_params = count_parameters(model_gen) print(" > Generator has {} parameters".format(num_params), flush=True) num_params = count_parameters(model_disc) @@ -526,9 +536,11 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = save_best_model(target_loss, best_loss, model_gen, + scheduler_gen, optimizer_gen, model_disc, optimizer_disc, + scheduler_disc, global_step, epoch, OUT_PATH, From dcfd62678480a1b3d02e9b91dbb22fa6d2c0ea8c Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:04:02 +0200 Subject: [PATCH 52/68] print lrg and lrD --- vocoder/utils/console_logger.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vocoder/utils/console_logger.py b/vocoder/utils/console_logger.py index ff7edfd0..50882160 100644 --- a/vocoder/utils/console_logger.py +++ b/vocoder/utils/console_logger.py @@ -15,8 +15,8 @@ tcolors = AttrDict({ class ConsoleLogger(): + # TODO: merge this with TTS ConsoleLogger def __init__(self): - # TODO: color code for value changes # use these to compare values between iterations self.old_train_loss_dict = None self.old_epoch_loss_dict = None @@ -36,7 +36,7 @@ class ConsoleLogger(): print(f"\n{tcolors.BOLD} > TRAINING ({self.get_time()}) {tcolors.ENDC}") def print_train_step(self, batch_steps, step, global_step, - step_time, loader_time, lr, + step_time, loader_time, lrG, lrD, loss_dict, avg_loss_dict): indent = " | > " print() @@ -48,7 +48,7 @@ class ConsoleLogger(): log_text += "{}{}: {:.5f} ({:.5f})\n".format(indent, key, value, avg_loss_dict[f'avg_{key}']) else: log_text += "{}{}: {:.5f} \n".format(indent, key, value) - log_text += f"{indent}step_time: {step_time:.2f}\n{indent}loader_time: {loader_time:.2f}\n{indent}lr: {lr:.5f}" + log_text += f"{indent}step_time: {step_time:.2f}\n{indent}loader_time: {loader_time:.2f}\n{indent}lrG: {lrG}\n{indent}lrD: {lrD}" print(log_text, flush=True) # pylint: disable=unused-argument From b445bcb9624905b58cbcac1196f708bb55e5a154 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:04:16 +0200 Subject: [PATCH 53/68] save schedulers with checkpoints --- vocoder/utils/io.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/vocoder/utils/io.py b/vocoder/utils/io.py index b6ea7484..9d350238 100644 --- a/vocoder/utils/io.py +++ b/vocoder/utils/io.py @@ -2,18 +2,27 @@ import os import torch import datetime -def save_model(model, optimizer, model_disc, optimizer_disc, current_step, - epoch, output_path, **kwargs): + +def save_model(model, optimizer, scheduler, model_disc, optimizer_disc, + scheduler_disc, current_step, epoch, output_path, **kwargs): model_state = model.state_dict() - model_disc_state = model_disc.state_dict() - optimizer_state = optimizer.state_dict() if optimizer is not None else None - optimizer_disc_state = optimizer_disc.state_dict( - ) if optimizer_disc is not None else None + model_disc_state = model_disc.state_dict()\ + if model_disc is not None else None + optimizer_state = optimizer.state_dict()\ + if optimizer is not None else None + optimizer_disc_state = optimizer_disc.state_dict()\ + if optimizer_disc is not None else None + scheduler_state = scheduler.state_dict()\ + if scheduler is not None else None + scheduler_disc_state = scheduler_disc.state_dict()\ + if scheduler_disc is not None else None state = { 'model': model_state, 'optimizer': optimizer_state, + 'scheduler': scheduler_state, 'model_disc': model_disc_state, 'optimizer_disc': optimizer_disc_state, + 'scheduler_disc': scheduler_disc_state, 'step': current_step, 'epoch': epoch, 'date': datetime.date.today().strftime("%B %d, %Y"), @@ -22,26 +31,29 @@ def save_model(model, optimizer, model_disc, optimizer_disc, current_step, torch.save(state, output_path) -def save_checkpoint(model, optimizer, model_disc, optimizer_disc, current_step, - epoch, output_folder, **kwargs): +def save_checkpoint(model, optimizer, scheduler, model_disc, optimizer_disc, + scheduler_disc, current_step, epoch, output_folder, + **kwargs): file_name = 'checkpoint_{}.pth.tar'.format(current_step) checkpoint_path = os.path.join(output_folder, file_name) print(" > CHECKPOINT : {}".format(checkpoint_path)) - save_model(model, optimizer, model_disc, optimizer_disc, current_step, - epoch, checkpoint_path, **kwargs) + save_model(model, optimizer, scheduler, model_disc, optimizer_disc, + scheduler_disc, current_step, epoch, checkpoint_path, **kwargs) -def save_best_model(target_loss, best_loss, model, optimizer, model_disc, - optimizer_disc, current_step, epoch, output_folder, - **kwargs): +def save_best_model(target_loss, best_loss, model, optimizer, scheduler, + model_disc, optimizer_disc, scheduler_disc, current_step, + epoch, output_folder, **kwargs): if target_loss < best_loss: file_name = 'best_model.pth.tar' checkpoint_path = os.path.join(output_folder, file_name) print(" > BEST MODEL : {}".format(checkpoint_path)) save_model(model, optimizer, + scheduler, model_disc, optimizer_disc, + scheduler_disc, current_step, epoch, checkpoint_path, From a1368e3eb741f9e4c9448fbab0b8bd936786ac02 Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:05:02 +0200 Subject: [PATCH 54/68] multiband melga nconfig --- vocoder/configs/multiband_melgan_config.json | 150 +++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 vocoder/configs/multiband_melgan_config.json diff --git a/vocoder/configs/multiband_melgan_config.json b/vocoder/configs/multiband_melgan_config.json new file mode 100644 index 00000000..3a5f0673 --- /dev/null +++ b/vocoder/configs/multiband_melgan_config.json @@ -0,0 +1,150 @@ +{ + "run_name": "multiband-melgan", + "run_description": "multiband melgan", + + // AUDIO PARAMETERS + "audio":{ + // stft parameters + "num_freq": 513, // number of stft frequency levels. Size of the linear spectogram frame. + "win_length": 1024, // stft window length in ms. + "hop_length": 256, // stft window hop-lengh in ms. + "frame_length_ms": null, // stft window length in ms.If null, 'win_length' is used. + "frame_shift_ms": null, // stft window hop-lengh in ms. If null, 'hop_length' is used. + + // Audio processing parameters + "sample_rate": 22050, // DATASET-RELATED: wav sample-rate. If different than the original data, it is resampled. + "preemphasis": 0.0, // pre-emphasis to reduce spec noise and make it more structured. If 0.0, no -pre-emphasis. + "ref_level_db": 20, // reference level db, theoretically 20db is the sound of air. + + // Silence trimming + "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) + "trim_db": 60, // threshold for timming silence. Set this according to your dataset. + + // Griffin-Lim + "power": 1.5, // value to sharpen wav signals after GL algorithm. + "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. + + // MelSpectrogram parameters + "num_mels": 80, // size of the mel spec frame. + "mel_fmin": 0.0, // minimum freq level for mel-spec. ~50 for male and ~95 for female voices. Tune for dataset!! + "mel_fmax": 8000.0, // maximum freq level for mel-spec. Tune for dataset!! + + // Normalization parameters + "signal_norm": true, // normalize spec values. Mean-Var normalization if 'stats_path' is defined otherwise range normalization defined by the other params. + "min_level_db": -100, // lower bound for normalization + "symmetric_norm": true, // move normalization to range [-1, 1] + "max_norm": 4.0, // scale normalization to range [-max_norm, max_norm] or [0, max_norm] + "clip_norm": true, // clip normalized values into the range. + "stats_path": null // DO NOT USE WITH MULTI_SPEAKER MODEL. scaler stats file computed by 'compute_statistics.py'. If it is defined, mean-std based notmalization is used and other normalization params are ignored + }, + + // DISTRIBUTED TRAINING + // "distributed":{ + // "backend": "nccl", + // "url": "tcp:\/\/localhost:54321" + // }, + + // MODEL PARAMETERS + "use_pqmf": true, + + // LOSS PARAMETERS + "use_stft_loss": true, + "use_subband_stft_loss": true, + "use_mse_gan_loss": true, + "use_hinge_gan_loss": false, + "use_feat_match_loss": false, // use only with melgan discriminators + + // loss weights + "stft_loss_weight": 0.5, + "subband_stft_loss_weight": 0.5, + "mse_G_loss_weight": 2.5, + "hinge_G_loss_weight": 2.5, + "feat_match_loss_weight": 25, + + // multiscale stft loss parameters + "stft_loss_params": { + "n_ffts": [1024, 2048, 512], + "hop_lengths": [120, 240, 50], + "win_lengths": [600, 1200, 240] + }, + + // subband multiscale stft loss parameters + "subband_stft_loss_params":{ + "n_ffts": [384, 683, 171], + "hop_lengths": [30, 60, 10], + "win_lengths": [150, 300, 60] + }, + + "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch + + // DISCRIMINATOR + "discriminator_model": "melgan_multiscale_discriminator", + "discriminator_model_params":{ + "base_channels": 16, + "max_channels":1024, + "downsample_factors":[4, 4, 4, 4] + }, + "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 + + // GENERATOR + "generator_model": "multiband_melgan_generator", + "generator_model_params": { + "upsample_factors":[8, 4, 2], + "num_res_blocks": 4 + }, + + // DATASET + "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", + "seq_len": 16384, + "pad_short": 2000, + "conv_pad": 0, + "use_noise_augment": false, + "use_cache": true, + + "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. + + // TRAINING + "batch_size": 64, // Batch size for training. Lower values than 32 might cause hard to learn attention. It is overwritten by 'gradual_training'. + + // VALIDATION + "run_eval": true, + "test_delay_epochs": 10, //Until attention is aligned, testing only wastes computation time. + "test_sentences_file": null, // set a file to load sentences to be used for testing. If it is null then we use default english sentences. + + // OPTIMIZER + "noam_schedule": false, // use noam warmup and lr schedule. + "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" + "warmup_steps_disc": 4000, + "epochs": 10000, // total number of epochs to train. + "wd": 0.0, // Weight decay weight. + "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 + "disc_clip_grad": -1, // Discriminator gradient clipping threshold. + "lr_scheduler_gen": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_gen_params": { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }, + "lr_scheduler_disc": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_disc_params": { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }, + "lr_gen": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_disc": 1e-4, + + // TENSORBOARD and LOGGING + "print_step": 25, // Number of steps to log traning on console. + "print_eval": false, // If True, it prints loss values for each step in eval run. + "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. + "checkpoint": true, // If true, it saves checkpoints per "save_step" + "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. + + // DATA LOADING + "num_loader_workers": 4, // number of training data loader processes. Don't set it too big. 4-8 are good values. + "num_val_loader_workers": 4, // number of evaluation data loader processes. + "eval_split_size": 10, + + // PATHS + "output_path": "/home/erogol/Models/LJSpeech/" +} + From 6d2cda97c8709f4697b03fb040f0b8afcb91c6ad Mon Sep 17 00:00:00 2001 From: erogol Date: Tue, 9 Jun 2020 23:14:10 +0200 Subject: [PATCH 55/68] q --- .../multiband-melgan_and_rwd_config.json | 52 +++++++++++-------- vocoder/train.py | 2 +- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/vocoder/configs/multiband-melgan_and_rwd_config.json b/vocoder/configs/multiband-melgan_and_rwd_config.json index f4b91aae..0b751854 100644 --- a/vocoder/configs/multiband-melgan_and_rwd_config.json +++ b/vocoder/configs/multiband-melgan_and_rwd_config.json @@ -1,6 +1,6 @@ { "run_name": "multiband-melgan-rwd", - "run_description": "multibadn melgan with random window discriminator", + "run_description": "multiband melgan with random window discriminator from https://arxiv.org/pdf/1909.11646.pdf", // AUDIO PARAMETERS "audio":{ @@ -54,33 +54,30 @@ "use_hinge_gan_loss": false, "use_feat_match_loss": false, // use only with melgan discriminators + // loss weights "stft_loss_weight": 0.5, "subband_stft_loss_weight": 0.5, "mse_G_loss_weight": 2.5, "hinge_G_loss_weight": 2.5, - "feat_match_loss_weight": 25.0, + "feat_match_loss_weight": 25, + // multiscale stft loss parameters "stft_loss_params": { "n_ffts": [1024, 2048, 512], "hop_lengths": [120, 240, 50], "win_lengths": [600, 1200, 240] }, + + // subband multiscale stft loss parameters "subband_stft_loss_params":{ "n_ffts": [384, 683, 171], "hop_lengths": [30, 60, 10], "win_lengths": [150, 300, 60] }, - "target_loss": "avg_G_loss", + + "target_loss": "avg_G_loss", // loss value to pick the best model to save after each epoch // DISCRIMINATOR - // "discriminator_model": "melgan_multiscale_discriminator", - // "discriminator_model_params":{ - // "base_channels": 16, - // "max_channels":1024, - // "downsample_factors":[4, 4, 4, 4] - // }, - "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 - "discriminator_model": "random_window_discriminator", "discriminator_model_params":{ "uncond_disc_donwsample_factors": [8, 4], @@ -88,6 +85,7 @@ "cond_disc_out_channels": [[128, 128, 256, 256], [128, 256, 256], [128, 256], [256], [128, 256]], "window_sizes": [512, 1024, 2048, 4096, 8192] }, + "steps_to_start_discriminator": 200000, // steps required to start GAN trainining.1 // GENERATOR "generator_model": "multiband_melgan_generator", @@ -97,11 +95,11 @@ }, // DATASET - "data_path": "/root/LJSpeech-1.1/wavs/", + "data_path": "/home/erogol/Data/LJSpeech-1.1/wavs/", "seq_len": 16384, "pad_short": 2000, "conv_pad": 0, - "use_noise_augment": true, + "use_noise_augment": false, "use_cache": true, "reinit_layers": [], // give a list of layer names to restore from the given checkpoint. If not defined, it reloads all heuristically matching layers. @@ -118,17 +116,27 @@ "noam_schedule": false, // use noam warmup and lr schedule. "warmup_steps_gen": 4000, // Noam decay steps to increase the learning rate from 0 to "lr" "warmup_steps_disc": 4000, - "epochs": 100000, // total number of epochs to train. - "wd": 0.000001, // Weight decay weight. - "lr_gen": 0.0001, // Initial learning rate. If Noam decay is active, maximum learning rate. - "lr_disc": 0.0001, - "gen_clip_grad": 10.0, - "disc_clip_grad": 10.0, + "epochs": 10000, // total number of epochs to train. + "wd": 0.0, // Weight decay weight. + "gen_clip_grad": -1, // Generator gradient clipping threshold. Apply gradient clipping if > 0 + "disc_clip_grad": -1, // Discriminator gradient clipping threshold. + "lr_scheduler_gen": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_gen_params": { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }, + "lr_scheduler_disc": "MultiStepLR", // one of the schedulers from https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate + "lr_scheduler_disc_params": { + "gamma": 0.5, + "milestones": [100000, 200000, 300000, 400000, 500000, 600000] + }, + "lr_gen": 1e-4, // Initial learning rate. If Noam decay is active, maximum learning rate. + "lr_disc": 1e-4, // TENSORBOARD and LOGGING "print_step": 25, // Number of steps to log traning on console. - "print_eval": false, // If True, it prints intermediate loss values in evalulation. - "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. + "print_eval": false, // If True, it prints loss values for each step in eval run. + "save_step": 25000, // Number of training steps expected to plot training stats on TB and save model checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" "tb_model_param_stats": false, // true, plots param stats per layer on tensorboard. Might be memory consuming, but good for debugging. @@ -138,6 +146,6 @@ "eval_split_size": 10, // PATHS - "output_path": "/data/rw/home/Trainings/" + "output_path": "/home/erogol/Models/LJSpeech/" } diff --git a/vocoder/train.py b/vocoder/train.py index c563dff0..a8a8f011 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -215,7 +215,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, torch.nn.utils.clip_grad_norm_(model_D.parameters(), c.disc_clip_grad) optimizer_D.step() - if c.scheduler_D is not None: + if scheduler_D is not None: scheduler_D.step() for key, value in loss_D_dict.items(): From 53f13461b921f861d5b07f2de28892e4704b96e1 Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 10 Jun 2020 13:45:43 +0200 Subject: [PATCH 56/68] better stats logging for TTS training --- train.py | 132 +++++++++++++++++++------------------------------------ 1 file changed, 45 insertions(+), 87 deletions(-) diff --git a/train.py b/train.py index 869557e6..51c73a9f 100644 --- a/train.py +++ b/train.py @@ -119,21 +119,7 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, verbose=(epoch == 0)) model.train() epoch_time = 0 - train_values = { - 'avg_postnet_loss': 0, - 'avg_decoder_loss': 0, - 'avg_stopnet_loss': 0, - 'avg_align_error': 0, - 'avg_step_time': 0, - 'avg_loader_time': 0 - } - if c.bidirectional_decoder: - train_values['avg_decoder_b_loss'] = 0 # decoder backward loss - train_values['avg_decoder_c_loss'] = 0 # decoder consistency loss - if c.ga_alpha > 0: - train_values['avg_ga_loss'] = 0 # guidede attention loss keep_avg = KeepAverage() - keep_avg.add_values(train_values) if use_cuda: batch_n_iter = int( len(data_loader.dataset) / (c.batch_size * num_gpus)) @@ -179,11 +165,6 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, mel_lengths, decoder_backward_output, alignments, alignment_lengths, alignments_backward, text_lengths) - if c.bidirectional_decoder: - keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_backward_loss'].item(), - 'avg_decoder_c_loss': loss_dict['decoder_c_loss'].item()}) - if c.ga_alpha > 0: - keep_avg.update_values({'avg_ga_loss': loss_dict['ga_loss'].item()}) # backward pass loss_dict['loss'].backward() @@ -193,7 +174,6 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, # compute alignment error (the lower the better ) align_error = 1 - alignment_diagonal_score(alignments) - keep_avg.update_value('avg_align_error', align_error) loss_dict['align_error'] = align_error # backpass and check the grad norm for stop loss @@ -208,23 +188,6 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, step_time = time.time() - start_time epoch_time += step_time - # update avg stats - update_train_values = { - 'avg_postnet_loss': float(loss_dict['postnet_loss'].item()), - 'avg_decoder_loss': float(loss_dict['decoder_loss'].item()), - 'avg_stopnet_loss': loss_dict['stopnet_loss'].item() \ - if isinstance(loss_dict['stopnet_loss'], float) else float(loss_dict['stopnet_loss'].item()), - 'avg_step_time': step_time, - 'avg_loader_time': loader_time - } - keep_avg.update_values(update_train_values) - - if global_step % c.print_step == 0: - c_logger.print_train_step(batch_n_iter, num_iter, global_step, - avg_spec_length, avg_text_length, - step_time, loader_time, current_lr, - loss_dict, keep_avg.avg_values) - # aggregate losses from processes if num_gpus > 1: loss_dict['postnet_loss'] = reduce_tensor(loss_dict['postnet_loss'].data, num_gpus) @@ -232,6 +195,30 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, loss_dict['loss'] = reduce_tensor(loss_dict['loss'] .data, num_gpus) loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) if c.stopnet else loss_dict['stopnet_loss'] + # detach loss values + loss_dict_new = dict() + for key, value in loss_dict.items(): + if isinstance(value, int) or isinstance(value, float): + loss_dict_new[key] = value + else: + loss_dict_new[key] = value.item() + loss_dict = loss_dict_new + + # update avg stats + update_train_values = dict() + for key, value in loss_dict.items(): + update_train_values['avg_' + key] = value + update_train_values['avg_loader_time'] = loader_time + update_train_values['avg_step_time'] = step_time + keep_avg.update_values(update_train_values) + + # print training progress + if global_step % c.print_step == 0: + c_logger.print_train_step(batch_n_iter, num_iter, global_step, + avg_spec_length, avg_text_length, + step_time, loader_time, current_lr, + loss_dict, keep_avg.avg_values) + if args.rank == 0: # Plot Training Iter Stats # reduce TB load @@ -266,7 +253,7 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, "alignment": plot_alignment(align_img), } - if c.bidirectional_decoder: + if c.bidirectional_decoder or c.double_decoder_consistency: figures["alignment_backward"] = plot_alignment(alignments_backward[0].data.cpu().numpy()) tb_logger.tb_train_figures(global_step, figures) @@ -286,16 +273,8 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, # Plot Epoch Stats if args.rank == 0: - # Plot Training Epoch Stats - epoch_stats = { - "loss_postnet": keep_avg['avg_postnet_loss'], - "loss_decoder": keep_avg['avg_decoder_loss'], - "stopnet_loss": keep_avg['avg_stopnet_loss'], - "alignment_error": keep_avg['avg_align_error'], - "epoch_time": epoch_time - } - if c.ga_alpha > 0: - epoch_stats['guided_attention_loss'] = keep_avg['avg_ga_loss'] + epoch_stats = {"epoch_time": epoch_time} + epoch_stats.update(keep_avg.avg_values) tb_logger.tb_train_epoch_stats(global_step, epoch_stats) if c.tb_model_param_stats: tb_logger.tb_model_weights(model, global_step) @@ -307,20 +286,7 @@ def evaluate(model, criterion, ap, global_step, epoch): data_loader = setup_loader(ap, model.decoder.r, is_val=True) model.eval() epoch_time = 0 - eval_values_dict = { - 'avg_postnet_loss': 0, - 'avg_decoder_loss': 0, - 'avg_stopnet_loss': 0, - 'avg_align_error': 0 - } - if c.bidirectional_decoder: - eval_values_dict['avg_decoder_b_loss'] = 0 # decoder backward loss - eval_values_dict['avg_decoder_c_loss'] = 0 # decoder consistency loss - if c.ga_alpha > 0: - eval_values_dict['avg_ga_loss'] = 0 # guidede attention loss keep_avg = KeepAverage() - keep_avg.add_values(eval_values_dict) - c_logger.print_eval_start() if data_loader is not None: for num_iter, data in enumerate(data_loader): @@ -352,11 +318,6 @@ def evaluate(model, criterion, ap, global_step, epoch): mel_lengths, decoder_backward_output, alignments, alignment_lengths, alignments_backward, text_lengths) - if c.bidirectional_decoder: - keep_avg.update_values({'avg_decoder_b_loss': loss_dict['decoder_b_loss'].item(), - 'avg_decoder_c_loss': loss_dict['decoder_c_loss'].item()}) - if c.ga_alpha > 0: - keep_avg.update_values({'avg_ga_loss': loss_dict['ga_loss'].item()}) # step time step_time = time.time() - start_time @@ -364,7 +325,7 @@ def evaluate(model, criterion, ap, global_step, epoch): # compute alignment score align_error = 1 - alignment_diagonal_score(alignments) - keep_avg.update_value('avg_align_error', align_error) + loss_dict['align_error'] = align_error # aggregate losses from processes if num_gpus > 1: @@ -373,14 +334,20 @@ def evaluate(model, criterion, ap, global_step, epoch): if c.stopnet: loss_dict['stopnet_loss'] = reduce_tensor(loss_dict['stopnet_loss'].data, num_gpus) - keep_avg.update_values({ - 'avg_postnet_loss': - float(loss_dict['postnet_loss'].item()), - 'avg_decoder_loss': - float(loss_dict['decoder_loss'].item()), - 'avg_stopnet_loss': - float(loss_dict['stopnet_loss'].item()), - }) + # detach loss values + loss_dict_new = dict() + for key, value in loss_dict.items(): + if isinstance(value, int) or isinstance(value, float): + loss_dict_new[key] = value + else: + loss_dict_new[key] = value.item() + loss_dict = loss_dict_new + + # update avg stats + update_train_values = dict() + for key, value in loss_dict.items(): + update_train_values['avg_' + key] = value + keep_avg.update_values(update_train_values) if c.print_eval: c_logger.print_eval_step(num_iter, loss_dict, keep_avg.avg_values) @@ -409,20 +376,11 @@ def evaluate(model, criterion, ap, global_step, epoch): c.audio["sample_rate"]) # Plot Validation Stats - epoch_stats = { - "loss_postnet": keep_avg['avg_postnet_loss'], - "loss_decoder": keep_avg['avg_decoder_loss'], - "stopnet_loss": keep_avg['avg_stopnet_loss'], - "alignment_error": keep_avg['avg_align_error'], - } - if c.bidirectional_decoder: - epoch_stats['loss_decoder_backward'] = keep_avg['avg_decoder_b_loss'] + if c.bidirectional_decoder or c.double_decoder_consistency: align_b_img = alignments_backward[idx].data.cpu().numpy() - eval_figures['alignment_backward'] = plot_alignment(align_b_img) - if c.ga_alpha > 0: - epoch_stats['guided_attention_loss'] = keep_avg['avg_ga_loss'] - tb_logger.tb_eval_stats(global_step, epoch_stats) + eval_figures['alignment2'] = plot_alignment(align_b_img) + tb_logger.tb_eval_stats(global_step, keep_avg.avg_values) tb_logger.tb_eval_figures(global_step, eval_figures) if args.rank == 0 and epoch > c.test_delay_epochs: From 1a061c4af5a407590d922cb809906670c7510a2c Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 10 Jun 2020 13:51:59 +0200 Subject: [PATCH 57/68] new config argument to plot tb figures at training --- config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.json b/config.json index 5e10c535..53b12fe1 100644 --- a/config.json +++ b/config.json @@ -104,7 +104,8 @@ "separate_stopnet": true, // Train stopnet seperately if 'stopnet==true'. It prevents stopnet loss to influence the rest of the model. It causes a better model, but it trains SLOWER. // TENSORBOARD and LOGGING - "print_step": 25, // Number of steps to log traning on console. + "print_step": 25, // Number of steps to log training on console. + "tb_plot_step:": 100, // Number of steps to plot TB training figures. "print_eval": false, // If True, it prints intermediate loss values in evalulation. "save_step": 10000, // Number of training steps expected to save traninpg stats and checkpoints. "checkpoint": true, // If true, it saves checkpoints per "save_step" From c24b57452da92f4e6bb819b38d218ed5861c5267 Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 10 Jun 2020 13:52:31 +0200 Subject: [PATCH 58/68] bug fix --- train.py | 5 ++--- utils/generic_utils.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 51c73a9f..02f28c1d 100644 --- a/train.py +++ b/train.py @@ -222,15 +222,14 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, if args.rank == 0: # Plot Training Iter Stats # reduce TB load - if global_step % 10 == 0: + if global_step % c.tb_plot_step == 0: iter_stats = { - "loss_posnet": loss_dict['postnet_loss'].item(), - "loss_decoder": loss_dict['decoder_loss'].item(), "lr": current_lr, "grad_norm": grad_norm, "grad_norm_st": grad_norm_st, "step_time": step_time } + iter_stats.update(loss_dict) tb_logger.tb_train_iter_stats(global_step, iter_stats) if global_step % c.save_step == 0: diff --git a/utils/generic_utils.py b/utils/generic_utils.py index eb6d21a9..a6398771 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -327,6 +327,7 @@ def check_config(c): # tensorboard _check_argument('print_step', c, restricted=True, val_type=int, min_val=1) + _check_argument('tb_plot_step', c, restricted=True, val_type=int, min_val=1) _check_argument('save_step', c, restricted=True, val_type=int, min_val=1) _check_argument('checkpoint', c, restricted=True, val_type=bool) _check_argument('tb_model_param_stats', c, restricted=True, val_type=bool) From 99420a2d9bbd91ac3ef1fe07a0b9b6af4fd40aea Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 10 Jun 2020 13:55:37 +0200 Subject: [PATCH 59/68] restore vocoder scheduler if it is in checkpoint --- vocoder/train.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vocoder/train.py b/vocoder/train.py index a8a8f011..091a6932 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -450,10 +450,14 @@ def main(args): # pylint: disable=redefined-outer-name weight_decay=0) # schedulers - scheduler_gen = getattr(torch.optim.lr_scheduler, c.lr_scheduler_gen) - scheduler_disc = getattr(torch.optim.lr_scheduler, c.lr_scheduler_disc) - scheduler_gen = scheduler_gen(optimizer_gen, **c.lr_scheduler_gen_params) - scheduler_disc = scheduler_disc(optimizer_disc, **c.lr_scheduler_disc_params) + scheduler_gen = None + scheduler_disc = None + if 'lr_scheduler_gen' in c: + scheduler_gen = getattr(torch.optim.lr_scheduler, c.lr_scheduler_gen) + scheduler_gen = scheduler_gen(optimizer_gen, **c.lr_scheduler_gen_params) + if 'lr_scheduler_disc' in c: + scheduler_disc = getattr(torch.optim.lr_scheduler, c.lr_scheduler_disc) + scheduler_disc = scheduler_disc(optimizer_disc, **c.lr_scheduler_disc_params) # setup criterion criterion_gen = GeneratorLoss(c) From 6a661f98e2778befc58d1c3da229ea629abb4e15 Mon Sep 17 00:00:00 2001 From: erogol Date: Wed, 10 Jun 2020 13:57:49 +0200 Subject: [PATCH 60/68] linter fix --- train.py | 4 ++-- vocoder/train.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/train.py b/train.py index 02f28c1d..f6d73a4a 100644 --- a/train.py +++ b/train.py @@ -198,7 +198,7 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, # detach loss values loss_dict_new = dict() for key, value in loss_dict.items(): - if isinstance(value, int) or isinstance(value, float): + if isinstance(value, (int, float)): loss_dict_new[key] = value else: loss_dict_new[key] = value.item() @@ -336,7 +336,7 @@ def evaluate(model, criterion, ap, global_step, epoch): # detach loss values loss_dict_new = dict() for key, value in loss_dict.items(): - if isinstance(value, int) or isinstance(value, float): + if isinstance(value, (int, float)): loss_dict_new[key] = value else: loss_dict_new[key] = value.item() diff --git a/vocoder/train.py b/vocoder/train.py index 091a6932..ce8f111a 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -17,7 +17,7 @@ from TTS.utils.generic_utils import (KeepAverage, count_parameters, from TTS.utils.io import copy_config_file, load_config from TTS.utils.radam import RAdam from TTS.utils.tensorboard_logger import TensorboardLogger -from TTS.utils.training import setup_torch_training_env, NoamLR +from TTS.utils.training import setup_torch_training_env from TTS.vocoder.datasets.gan_dataset import GANDataset from TTS.vocoder.datasets.preprocess import load_wav_data # from distribute import (DistributedSampler, apply_gradient_allreduce, From 4d1d04a6a03a270e867c12ece86418988867bee9 Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 11 Jun 2020 10:48:20 +0200 Subject: [PATCH 61/68] bug fix --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index f6d73a4a..8581132a 100644 --- a/train.py +++ b/train.py @@ -237,7 +237,7 @@ def train(model, criterion, optimizer, optimizer_st, scheduler, # save model save_checkpoint(model, optimizer, global_step, epoch, model.decoder.r, OUT_PATH, optimizer_st=optimizer_st, - model_loss=loss_dict['postnet_loss'].item()) + model_loss=loss_dict['postnet_loss']) # Diagnostic visualizations const_spec = postnet_output[0].data.cpu().numpy() From 92ed9b90c3de90523be1898145b99722acc80fea Mon Sep 17 00:00:00 2001 From: erogol Date: Thu, 11 Jun 2020 12:42:13 +0200 Subject: [PATCH 62/68] reformatting vocoder training and disable G_adv eval if step < d_training --- vocoder/train.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/vocoder/train.py b/vocoder/train.py index ce8f111a..74b3e759 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -112,7 +112,6 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, ############################## # generator pass - optimizer_G.zero_grad() y_hat = model_G(c_G) y_hat_sub = None y_G_sub = None @@ -121,8 +120,8 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, # PQMF formatting if y_hat.shape[1] > 1: y_hat_sub = y_hat - y_hat = model_G.pqmf_synthesis(y_hat) y_hat_vis = y_hat + y_hat = model_G.pqmf_synthesis(y_hat) y_G_sub = model_G.pqmf_analysis(y_G) if global_step > c.steps_to_start_discriminator: @@ -142,12 +141,11 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, if isinstance(D_out_fake, tuple): scores_fake, feats_fake = D_out_fake if D_out_real is None: - scores_real, feats_real = None, None + feats_real = None else: - scores_real, feats_real = D_out_real + _, feats_real = D_out_real else: scores_fake = D_out_fake - scores_real = D_out_real else: scores_fake, feats_fake, feats_real = None, None, None @@ -157,6 +155,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, loss_G = loss_G_dict['G_loss'] # optimizer generator + optimizer_G.zero_grad() loss_G.backward() if c.gen_clip_grad > 0: torch.nn.utils.clip_grad_norm_(model_G.parameters(), @@ -184,8 +183,6 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, if y_hat.shape[1] > 1: y_hat = model_G.pqmf_synthesis(y_hat) - optimizer_D.zero_grad() - # run D with or without cond. features if len(signature(model_D.forward).parameters) == 2: D_out_fake = model_D(y_hat.detach(), c_D) @@ -210,6 +207,7 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, loss_D = loss_D_dict['D_loss'] # optimizer discriminator + optimizer_D.zero_grad() loss_D.backward() if c.disc_clip_grad > 0: torch.nn.utils.clip_grad_norm_(model_D.parameters(), @@ -326,25 +324,30 @@ def evaluate(model_G, criterion_G, model_D, criterion_D, ap, global_step, epoch) y_hat = model_G.pqmf_synthesis(y_hat) y_G_sub = model_G.pqmf_analysis(y_G) - if len(signature(model_D.forward).parameters) == 2: - D_out_fake = model_D(y_hat, c_G) - else: - D_out_fake = model_D(y_hat) - D_out_real = None - if c.use_feat_match_loss: - with torch.no_grad(): - D_out_real = model_D(y_G) + if global_step > c.steps_to_start_discriminator: - # format D outputs - if isinstance(D_out_fake, tuple): - scores_fake, feats_fake = D_out_fake - if D_out_real is None: - feats_real = None + if len(signature(model_D.forward).parameters) == 2: + D_out_fake = model_D(y_hat, c_G) else: - _, feats_real = D_out_real + D_out_fake = model_D(y_hat) + D_out_real = None + + if c.use_feat_match_loss: + with torch.no_grad(): + D_out_real = model_D(y_G) + + # format D outputs + if isinstance(D_out_fake, tuple): + scores_fake, feats_fake = D_out_fake + if D_out_real is None: + feats_real = None + else: + _, feats_real = D_out_real + else: + scores_fake = D_out_fake else: - scores_fake = D_out_fake + scores_fake, feats_fake, feats_real = None, None, None # compute losses loss_G_dict = criterion_G(y_hat, y_G, scores_fake, feats_fake, From 1c2dc9f73909590aed91734cc1ea0a83ee1fb97c Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 11:12:57 +0200 Subject: [PATCH 63/68] enable loading precomputed vocoder dataset --- vocoder/datasets/gan_dataset.py | 25 +++++++++++++++++++------ vocoder/datasets/preprocess.py | 21 +++++++++++++++++++++ vocoder/train.py | 4 ++-- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/vocoder/datasets/gan_dataset.py b/vocoder/datasets/gan_dataset.py index 5e410151..55513e7d 100644 --- a/vocoder/datasets/gan_dataset.py +++ b/vocoder/datasets/gan_dataset.py @@ -28,6 +28,7 @@ class GANDataset(Dataset): self.ap = ap self.item_list = items + self.compute_feat = not isinstance(items[0], (tuple, list)) self.seq_len = seq_len self.hop_len = hop_len self.pad_short = pad_short @@ -77,14 +78,26 @@ class GANDataset(Dataset): def load_item(self, idx): """ load (audio, feat) couple """ - wavpath = self.item_list[idx] - # print(wavpath) + if self.compute_feat: + # compute features from wav + wavpath = self.item_list[idx] + # print(wavpath) - if self.use_cache and self.cache[idx] is not None: - audio, mel = self.cache[idx] + if self.use_cache and self.cache[idx] is not None: + audio, mel = self.cache[idx] + else: + audio = self.ap.load_wav(wavpath) + mel = self.ap.melspectrogram(audio) else: - audio = self.ap.load_wav(wavpath) - mel = self.ap.melspectrogram(audio) + + # load precomputed features + wavpath, feat_path = self.item_list[idx] + + if self.use_cache and self.cache[idx] is not None: + audio, mel = self.cache[idx] + else: + audio = self.ap.load_wav(wavpath) + mel = np.load(feat_path) if len(audio) < self.seq_len + self.pad_short: audio = np.pad(audio, (0, self.seq_len + self.pad_short - len(audio)), \ diff --git a/vocoder/datasets/preprocess.py b/vocoder/datasets/preprocess.py index 01e01e3e..be60c13a 100644 --- a/vocoder/datasets/preprocess.py +++ b/vocoder/datasets/preprocess.py @@ -1,5 +1,6 @@ import glob import os +from pathlib import Path import numpy as np @@ -9,8 +10,28 @@ def find_wav_files(data_path): return wav_paths +def find_feat_files(data_path): + feat_paths = glob.glob(os.path.join(data_path, '**', '*.npy'), recursive=True) + return feat_paths + + def load_wav_data(data_path, eval_split_size): wav_paths = find_wav_files(data_path) np.random.seed(0) np.random.shuffle(wav_paths) return wav_paths[:eval_split_size], wav_paths[eval_split_size:] + + +def load_wav_feat_data(data_path, feat_path, eval_split_size): + wav_paths = sorted(find_wav_files(data_path)) + feat_paths = sorted(find_feat_files(feat_path)) + assert len(wav_paths) == len(feat_paths) + for wav, feat in zip(wav_paths, feat_paths): + wav_name = Path(wav).stem + feat_name = Path(feat).stem + assert wav_name == feat_name + + items = list(zip(wav_paths, feat_paths)) + np.random.seed(0) + np.random.shuffle(items) + return items[:eval_split_size], items[eval_split_size:] diff --git a/vocoder/train.py b/vocoder/train.py index 74b3e759..4d1b1029 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -19,7 +19,7 @@ from TTS.utils.radam import RAdam from TTS.utils.tensorboard_logger import TensorboardLogger from TTS.utils.training import setup_torch_training_env from TTS.vocoder.datasets.gan_dataset import GANDataset -from TTS.vocoder.datasets.preprocess import load_wav_data +from TTS.vocoder.datasets.preprocess import load_wav_data, load_wav_feat_data # from distribute import (DistributedSampler, apply_gradient_allreduce, # init_distributed, reduce_tensor) from TTS.vocoder.layers.losses import DiscriminatorLoss, GeneratorLoss @@ -543,8 +543,8 @@ def main(args): # pylint: disable=redefined-outer-name best_loss = save_best_model(target_loss, best_loss, model_gen, - scheduler_gen, optimizer_gen, + scheduler_gen, model_disc, optimizer_disc, scheduler_disc, From 9af6bb0c297511c31973c898a8cd4da263a70849 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 11:37:32 +0200 Subject: [PATCH 64/68] bug fix --- vocoder/utils/console_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vocoder/utils/console_logger.py b/vocoder/utils/console_logger.py index 50882160..4f44986c 100644 --- a/vocoder/utils/console_logger.py +++ b/vocoder/utils/console_logger.py @@ -84,7 +84,7 @@ class ConsoleLogger(): color = '' sign = '+' diff = 0 - if self.old_eval_loss_dict is not None: + if self.old_eval_loss_dict is not None and key in self.old_eval_loss_dict: diff = value - self.old_eval_loss_dict[key] if diff < 0: color = tcolors.OKGREEN From 546186bbf1f62c2d36c982eb01d85bbdd8f09ec9 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 11:54:54 +0200 Subject: [PATCH 65/68] visualization bug fix --- vocoder/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vocoder/train.py b/vocoder/train.py index 4d1b1029..326bd90e 100644 --- a/vocoder/train.py +++ b/vocoder/train.py @@ -120,8 +120,8 @@ def train(model_G, criterion_G, optimizer_G, model_D, criterion_D, optimizer_D, # PQMF formatting if y_hat.shape[1] > 1: y_hat_sub = y_hat - y_hat_vis = y_hat y_hat = model_G.pqmf_synthesis(y_hat) + y_hat_vis = y_hat y_G_sub = model_G.pqmf_analysis(y_G) if global_step > c.steps_to_start_discriminator: From 0b78977662bfa9d59da4c2f9ce8d4647e093c3db Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 12:11:23 +0200 Subject: [PATCH 66/68] more comments to audio.py --- utils/audio.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/audio.py b/utils/audio.py index 13eab3d6..f941f609 100644 --- a/utils/audio.py +++ b/utils/audio.py @@ -57,8 +57,10 @@ class AudioProcessor(object): self.stats_path = stats_path # setup stft parameters if hop_length is None: + # compute stft parameters from given time values self.n_fft, self.hop_length, self.win_length = self._stft_parameters() else: + # use stft parameters from config file self.hop_length = hop_length self.win_length = win_length self.n_fft = (self.num_freq - 1) * 2 From 3eb730acf0a660d96f3af78cde0f5379b72fb3c6 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 19:32:49 +0200 Subject: [PATCH 67/68] update vocoder loss implemenatations and fix MSEDLoss --- vocoder/layers/losses.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/vocoder/layers/losses.py b/vocoder/layers/losses.py index a24b129c..4774ab4e 100644 --- a/vocoder/layers/losses.py +++ b/vocoder/layers/losses.py @@ -88,7 +88,7 @@ class MSEGLoss(nn.Module): """ Mean Squared Generator Loss """ # pylint: disable=no-self-use def forward(self, score_fake): - loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) + loss_fake = F.mse_loss(score_fake, score_fake.new_ones(score_fake.shape)) return loss_fake @@ -96,7 +96,8 @@ class HingeGLoss(nn.Module): """ Hinge Discriminator Loss """ # pylint: disable=no-self-use def forward(self, score_fake): - loss_fake = torch.mean(F.relu(1. + score_fake)) + # TODO: this might be wrong + loss_fake = torch.mean(F.relu(1. - score_fake)) return loss_fake @@ -107,10 +108,14 @@ class HingeGLoss(nn.Module): class MSEDLoss(nn.Module): """ Mean Squared Discriminator Loss """ + def __init__(self,): + super(MSEDLoss, self).__init__() + self.loss_func = nn.MSELoss() + # pylint: disable=no-self-use def forward(self, score_fake, score_real): - loss_real = torch.mean(torch.sum(torch.pow(score_real - 1.0, 2), dim=[1, 2])) - loss_fake = torch.mean(torch.sum(torch.pow(score_fake, 2), dim=[1, 2])) + loss_real = self.loss_func(score_real, score_real.new_ones(score_real.shape)) + loss_fake = self.loss_func(score_fake, score_fake.new_zeros(score_fake.shape)) loss_d = loss_real + loss_fake return loss_d, loss_real, loss_fake @@ -126,11 +131,15 @@ class HingeDLoss(nn.Module): class MelganFeatureLoss(nn.Module): + def __init__(self,): + super(MelganFeatureLoss, self).__init__() + self.loss_func = nn.L1Loss() + # pylint: disable=no-self-use def forward(self, fake_feats, real_feats): loss_feats = 0 for fake_feat, real_feat in zip(fake_feats, real_feats): - loss_feats += torch.mean(torch.abs(fake_feat - real_feat)) + loss_feats += self.loss_func(fake_feats, real_feats) loss_feats /= len(fake_feats) + len(real_feats) return loss_feats From 773e9c7a72ab7fc930ad0e26cafbb660fcdb82b2 Mon Sep 17 00:00:00 2001 From: erogol Date: Fri, 12 Jun 2020 19:50:07 +0200 Subject: [PATCH 68/68] remove redundant params in vocoder config file --- vocoder/configs/multiband_melgan_config.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/vocoder/configs/multiband_melgan_config.json b/vocoder/configs/multiband_melgan_config.json index 3a5f0673..164fe037 100644 --- a/vocoder/configs/multiband_melgan_config.json +++ b/vocoder/configs/multiband_melgan_config.json @@ -20,9 +20,6 @@ "do_trim_silence": true,// enable trimming of slience of audio as you load it. LJspeech (false), TWEB (false), Nancy (true) "trim_db": 60, // threshold for timming silence. Set this according to your dataset. - // Griffin-Lim - "power": 1.5, // value to sharpen wav signals after GL algorithm. - "griffin_lim_iters": 60,// #griffin-lim iterations. 30-60 is a good range. Larger the value, slower the generation. // MelSpectrogram parameters "num_mels": 80, // size of the mel spec frame.