From 7304978be3491cf5c5b6f58c762506af097a66f3 Mon Sep 17 00:00:00 2001 From: Eren Golge Date: Thu, 1 Feb 2018 08:26:40 -0800 Subject: [PATCH] new lr schedule --- config.json | 3 ++- layers/.attention.py.swp | Bin 12288 -> 0 bytes layers/.tacotron.py.swp | Bin 16384 -> 0 bytes models/.tacotron.py.swp | Bin 12288 -> 0 bytes models/tacotron.py | 2 ++ train.py | 28 +++++++++++++++++----------- utils/generic_utils.py | 7 +++++++ 7 files changed, 28 insertions(+), 12 deletions(-) delete mode 100644 layers/.attention.py.swp delete mode 100644 layers/.tacotron.py.swp delete mode 100644 models/.tacotron.py.swp diff --git a/config.json b/config.json index aaf36669..d3f5e5ed 100644 --- a/config.json +++ b/config.json @@ -16,9 +16,10 @@ "lr_patience": 5, "lr_decay": 0.5, "batch_size": 256, + "r": 5, + "griffin_lim_iters": 60, "power": 1.5, - "r": 5, "num_loader_workers": 32, diff --git a/layers/.attention.py.swp b/layers/.attention.py.swp deleted file mode 100644 index 8fd631d72f0ae48d75896509347f61c6412c96ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2O^6&t6vrzOKN7R?>tY18+(GHV+1^ZyCJxLYQCCE>k+?=MgrR1pW~NPd^`yFb zcPHSAcoR<=(VK{#RfvZuf;V#%k0N*rn491sdh=gZ{WY_*(HsNS@T0f8U%z_wf3IFm zNOupOe(3}~;vXhB?j_{+KhD2-@rg&sm6Q-J<6#`?2RA;Vu-Ck?g?+kZ9WIY${1)%2 zWO;4vwdIgaxJ;IrQe3DY7XEnhe|$L`r+`!7CKO23VCm=rdFoK}pnj{i_!xcQ{?~89 zfiraqI0c*nP64NYQ@|fTK0`G(4;9+q6E<%0;Ti|VQ z4D1Daz@K*#@-4UoJ_GN8Bj6cu09?C+kT1a(U<*7A4uku^b?EpVxC}lAn_vU10Il)k zz#UEjr@*bG0L>1`RVoGT@i6SfsT!xMLGk4h`H$!J%O;qH!B9lFLjTN;;=3o>)}FF& zppx?`=NGuwtQUqg>UEjwjT*EcMD_U&R4jWVzZ*qfF_NE((piH>Jc{LHp4VpPI1j8w zkLBUeUMv*fRCFC*ZN;)Oc0L~jzv3tHKt*h`9NTzoFE#75R-BW+F^o`Cs z3sYXsSUb`85wbq{>jB^JAh(?O35NF$)*BQd6K$uDU|`LIgo;?v)mZRW8B!4r+?j!k z9Cjk!8!-_ikvCN?`1q~IlLSHUoC~PfQ9)FR8rjITE(fpp&4-slbo6aIW*|(38oNX> zHRW)H^F9p}RWUuax;o?4ya}haqIyhuRwGmHwn%4ms?q0S0YhqEHHuOu`gAP$dJv~c zC9zpOm1An5v!o2UvUwZGIO>F44Am&{tX6sb=C`M_WJ)`KL@W6+VHo$gX%nai%#YcO z=*lp@NN#}|*k&}>m40E?vaLr%OrlZ}dXz|XRQl5S-FE-fbf0hR6m`kr{A#X@F#tpkV>wc#b zh(L8ZUYS81j5%gQZ7&VN7zMZ(h{0mLB2?kTDwT&&5vyE-4blr)l9W9y3&<0~KN0uS zki)T+G3)VLI$

$z<t?;~cPDUI_b=ev*<7y0#lRF#(im-&5I(SsUIcQm{MC#`Bfglj?Ha0B)t^#mu+&9 zWh~TJ0>PQ|FiWJULAwD<+AAzfD%+Yd*F|98SkvWDX2v|NWNy1#<9n`S)6b5xQJ)Ex z7JY4phNh=F)1LINwpP}Kwz-*ULuSF*tyc&dMB^BRFi1sDe<067$`ZOlN~7FDh{{c$ Yr79juhCSZCf11gFbwf_FYvZ5%3x%4g&Hw-a diff --git a/layers/.tacotron.py.swp b/layers/.tacotron.py.swp deleted file mode 100644 index 631a4a89d57c71724aecf5f142b44d0bef4e7000..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOON<;x8LnU_5FijcNIoE-vLob~cy@MY?SQ>xz+R(djUPChu`L)@qu%M7nfCT{ zkNV+V`(R#25RMfj&OvgbB#=W8AbcRl1SBAdxdl$hoH>L@L`d-c_2}xJon34sgh1WW zw>{HU^*_G)>#s-8uFao6ahiX&afso1kg=P$hF^X6)1PForHqLv9E3gMi=vmrbM5w1bG~y~MDbkW^uiPKK*>PKK*>PKK*>PKK*_-WI0MOe7yCMVc%SxTsPFeo z-2b7ki~6}YQNBxmKcVlpCd&2K@>ViXGEg#5GEg#5GEg#5GEg#5GEg#5GEg#5GVotu zz;PM70KE^<0RZ0roBjV^KFZjifLDPRffs-PI0hUA_5*+Y2xGqmZUEl^&I9{^zdgX% zTfm#ZE5J2i1$Ywp9Iyx+0rmlJeweYBfvdo2;A6n+A7bqLz>`27H~{Pc-u)o-0WSg9 zfv16MKni>bm;v6opRrrO4d7dV3mgT$0C3>F4?rjIJn$TF1Na`W0xSRwc7`M)Zfv7E76=a+8!N{v~_`T*N zycvqT+p29blIfH5#l<+_Lh;Y?7?%>+rx z)EMgK9NeB`YY=rrN4AfIpRV#pcomVA=efpbd9%h3@D7FAYNph1N&-%zoeG6jRLq&s zSwt_{*pZ4x&SrNUhCT+o46oD9=2`i*ns;;|=P1v(ERrX~ZQX_&s6Gk(%`KgoXPiD8 zdFy;`jzg2>qm^+{R5h)ikdSNu=51fk*`D#0?<7ie|7r+Oe<*@h#Hn*tYmOA8*habf zV6~)3MAhR<%TK@tq>Wmmi`3#J-L95Ec_n*9HLMFKTg}fDMo6pETsVyMv2b|Rq?nnR znM~I#U&`kYK5!D`LgbV*ibRm`6nbqQpXEAf@IWLRVFUsWpmOZFspC`H3{*18w`k6~ zGNFckN~2Ux5ULz@kdu5QGsq%$a(V-iMi8mhK){Hyzstk^BJ2519CK<~MP^g^hkVPN zibRq|0bfQ$w_%`4x-=}Bjd_fOw(SR+d1PP2-e44ZDor=q7{QdMYqsLDsYqQalX+vg zSZljvZ&>HJ@~Tu^Jf-zoO|s6;fzxhKs%*GU;xzi6?|0?2Nebwa0?09tqf@RP=QamU zs9Vknyn^3fnn26Mk^6ThPR6c|lDi7uVv+`Hs++Z_oqJlkycSe9tzTq1P1Ul<=7xL~ zEFLCGxfk;wOgLppyL`yRsPt?^Z0+#PEq!ej!xa04k#%%w!xg*PG=eFFCM%hnM&StK zFden#8%K`RxfsW-rkFiw9j)-kL*EZKFi-JB1aTPk@oj(1VV{%w2@+-Gj6{Op&oZfV zhd+8+Qa7X#J^krBtvBopy@ME8R7j?F_M1B6DgW*4DLpq7Frz7!v6Zo;s&5lxrj5K? z;>}Wv9`BGvjcT5A!psL5Ep;8C69(-Zse0weq+R-t&QCg4-6$MGzsfN(+xTkTWO9=2 zdRpJ8LYhBiEU6z#tJvKdpjt$3sVN(2m8uuMsP)e`9<`cewQZA<^ke#BD_dyk6oloQ zT*XaJ@0~l#F0{H=E-GWEEk2DV)aa<0l@>Hot6It!?o~XU#XKjHZNg}W7?A>7orOm+ z-|E>_Wh^DI7zX_)9Px&eV1UX=(8F{VC43-~?#PMPyHswr18kehOU}@B0w+CYq||x# z@j~Q`v1q_fkrX9i)EjcGBM5R28%~;pgUE4B>3JveoHbuCU0XI~6->Ca@LlBc2f>W? z|9`~3^b^>_(*FO1-c~)KLBXoe*}0CcpLls=Yekmr+|~d$AQ0NFaHMcJK!ep zL*QB98DIo_1vn18hyDF+fcE@92Cf5-0qV_2Z*mv0p(RPP%=<5@PEkwO{aRtMax!foaD}gKh~nvYOUe;KK89N zlX`Ji1a3F%W4^AyFl`g3Bx2d%_}(C(wQ%k%P8VXPm-AL5#y^OyYpKv| zPYvfWN7ouyzbD=x4bwQc6}NIH#Mpu@l9*XSRQM7!(%In}odI!qIA=Ga*ry7b<|7<2 zUgoq~!?}z$x<<#9?XD{@)zZvuMCR{nk#N?xlwjYI@XzNO+pP*U7C1mFJ~1Mvi=)zU zkoO%%nsg)Kc#_<{!cO+p9*R*-6IfJ5ea@YL z*$RjlxiY5HvThP~N5a?LTcoH0f`#k#5{?@D5)MhiDCTmlrg<_Ci^eL|&|=t>3;1Fk zvL-90F@gb@YY_zjPGMaaJHw9WZqiaXt4&oIwa;G_zA|~B6NCZFS23Rpg6f&vNtHz| zsIh4{S<~>DD_2$*&uM9^0blH)Aq^|FlTo9o3&A+ODMm?!pQUr7WEh3%V2HagcJP%m zXL4D^xTkUo-fdK?*~*<|_Dr2~}q1TvC~I30!2<<{hbI;YJ@i~baSAV!R{(vK;=BSV{w zNuDHT*rfZ^8kyv*G?(lRVObf;o+Rp+QA{0KdXZt%kW9fC>O^uUp|kg5cF`$r-p1#~(DMbSmpb;@q$)ZB_8cEa!TDiJV;%pi zNw(3I?~WBl_JnWTM!ggF!Vr$qDA#8N$B&=q=O>m%>)H6q8<*&d z-U|fRCLve1uf6}}bEnBqB_T}k1y8GvyI)xv?cUv@>HV~ExKjxJA&ZpUnNHu`$#~3C zxue2}E5UQGSp6SY*2WUB1n!4Gspi`UC&}JqcU%qII`cGr>dE)+$Ah)C1S|ndz!I?@6eCXx6n=K73gK?8R)mi3Hb`T0ZpNOXcM}1 zl911#06KzBLAM?w&Yk<*N&0WVci zD(TWR$yq46c9m0M@x%XE6$<)=05~FA7#HDk=J{smzkJXiqajzb| ze$fnZ>6`w6TNPb(NI2oS^si3#&$Vm)K`+zVjV=$kc}*9Ju6&pF%Az*a>j2a9P1QQg zpuA0dZJ&3{lEqSaai~Hs=OPQ!z!|%A?CtdzZz@*RHzLuUGC`$^+sdhm?9Nva)~gvL z@pZrV-1%;TH=#D5&zq)$&&!dkY{pYLa%mfNm61!#thI}^!iA%?H$!v1aaHT+o4pW? zp^=6S3;on}mjx5fRG;i_cBPRQ1WBH#AaJ_$OpNQ%_!mNas;Ifvz&x0W@^{CReXO&O zKRt3y{no>(if^Z^AgmGc>)Prv)gE;&{=wKK5jJy4O&$)4{1T7Lv Embedding dim : {}".format(len(symbols))) + # Trying smaller std self.embedding.weight.data.normal_(0, 0.3) self.encoder = Encoder(embedding_dim) diff --git a/train.py b/train.py index 2776c1cf..72d58d19 100644 --- a/train.py +++ b/train.py @@ -19,7 +19,7 @@ from tensorboardX import SummaryWriter from utils.generic_utils import (Progbar, remove_experiment_folder, create_experiment_folder, save_checkpoint, - load_config) + load_config, lr_decay) from utils.model import get_param_size from datasets.LJSpeech import LJSpeechDataset from models.tacotron import Tacotron @@ -97,10 +97,11 @@ def main(args): else: criterion = nn.L1Loss() - n_priority_freq = int(3000 / (c.sample_rate * 0.5) * c.num_freq) +n_priority_freq = int(3000 / (c.sample_rate * 0.5) * c.num_freq) - lr_scheduler = ReduceLROnPlateau(optimizer, factor=c.lr_decay, - patience=c.lr_patience, verbose=True) + #lr_scheduler = ReduceLROnPlateau(optimizer, factor=c.lr_decay, + # patience=c.lr_patience, verbose=True) + epoch_time = 0 for epoch in range(c.epochs): @@ -119,14 +120,19 @@ def main(args): current_step = i + args.restore_step + epoch * len(dataloader) + 1 + # setup lr + current_lr = lr_decay(init_lr, current_step) + for params_group in optimizer.param_groups: + param_group['lr'] = current_lr + optimizer.zero_grad() - try: - mel_input = np.concatenate((np.zeros( - [c.batch_size, 1, c.num_mels], dtype=np.float32), - mel_input[:, 1:, :]), axis=1) - except: - raise TypeError("not same dimension") + #try: + # mel_input = np.concatenate((np.zeros( + # [c.batch_size, 1, c.num_mels], dtype=np.float32), + # mel_input[:, 1:, :]), axis=1) + #except: + # raise TypeError("not same dimension") if use_cuda: text_input_var = Variable(torch.from_numpy(text_input).type( @@ -204,7 +210,7 @@ def main(args): tb.add_image('Spec/Reconstruction', const_spec, current_step) tb.add_image('Spec/GroundTruth', gt_spec, current_step) - lr_scheduler.step(loss.data[0]) + #lr_scheduler.step(loss.data[0]) tb.add_scalar('Time/EpochTime', epoch_time, epoch) epoch_time = 0 diff --git a/utils/generic_utils.py b/utils/generic_utils.py index aa329fb7..06e884ce 100644 --- a/utils/generic_utils.py +++ b/utils/generic_utils.py @@ -52,6 +52,13 @@ def save_checkpoint(state, filename='checkpoint.pth.tar'): torch.save(state, filename) +def lr_decay(init_lr, global_step): + warmup_steps = 4000.0 + step = global_step + 1. + lr = init_lr * warmup_steps**0.5 * np.minimum(step * warmup_steps**-1.5, + step**-0.5) + return lr + class Progbar(object): """Displays a progress bar. # Arguments