Skip to content

Commit 78a1dfb

Browse files
author
Andy Zhang
committed
Got FC network stable on 3 bigger layers
1 parent 1a39ffc commit 78a1dfb

File tree

1 file changed

+210
-66
lines changed

1 file changed

+210
-66
lines changed

‎tensorflow/examples/udacity/3_regularization.ipynb‎

Lines changed: 210 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
},
2121
{
2222
"cell_type": "code",
23-
"execution_count": 2,
23+
"execution_count": 1,
2424
"metadata": {
2525
"cellView": "both",
2626
"colab": {
@@ -54,7 +54,7 @@
5454
},
5555
{
5656
"cell_type": "code",
57-
"execution_count": 3,
57+
"execution_count": 2,
5858
"metadata": {
5959
"cellView": "both",
6060
"colab": {
@@ -131,7 +131,7 @@
131131
},
132132
{
133133
"cell_type": "code",
134-
"execution_count": 4,
134+
"execution_count": 3,
135135
"metadata": {
136136
"cellView": "both",
137137
"colab": {
@@ -196,7 +196,7 @@
196196
},
197197
{
198198
"cell_type": "code",
199-
"execution_count": 5,
199+
"execution_count": 4,
200200
"metadata": {
201201
"cellView": "both",
202202
"colab": {
@@ -589,75 +589,220 @@
589589
"cell_type": "code",
590590
"execution_count": 12,
591591
"metadata": {
592-
"collapsed": false
592+
"collapsed": false,
593+
"scrolled": false
593594
},
594595
"outputs": [
595596
{
596597
"name": "stdout",
597598
"output_type": "stream",
598599
"text": [
599600
"Initialized\n",
600-
"Minibatch loss at step 0 : 34718.1\n",
601-
"Minibatch accuracy: 15.6%\n",
602-
"Validation accuracy: 9.4%\n",
603-
"Minibatch loss at step 1000 : 2.31229\n",
604-
"Minibatch accuracy: 10.9%\n",
605-
"Validation accuracy: 10.4%\n",
606-
"Minibatch loss at step 2000 : 2.3071\n",
607-
"Minibatch accuracy: 9.4%\n",
608-
"Validation accuracy: 10.4%\n",
609-
"Minibatch loss at step 3000 : 2.29897\n",
610-
"Minibatch accuracy: 12.5%\n",
611-
"Validation accuracy: 10.4%\n",
612-
"Minibatch loss at step 4000 : 2.30373\n",
613-
"Minibatch accuracy: 7.0%\n",
614-
"Validation accuracy: 10.4%\n",
615-
"Minibatch loss at step 5000 : 2.30152\n",
601+
"Minibatch loss at step 0 : 58.3308\n",
616602
"Minibatch accuracy: 9.4%\n",
617-
"Validation accuracy: 10.4%\n",
618-
"Minibatch loss at step 6000 : 2.30267\n",
619-
"Minibatch accuracy: 12.5%\n",
620-
"Validation accuracy: 10.1%\n",
621-
"Minibatch loss at step 7000 : 2.30219\n",
622-
"Minibatch accuracy: 15.6%\n",
623-
"Validation accuracy: 10.4%\n",
624-
"Minibatch loss at step 8000 : 2.3031\n",
625-
"Minibatch accuracy: 10.9%\n",
626-
"Validation accuracy: 10.1%\n",
627-
"Minibatch loss at step 9000 : 2.30252\n",
628-
"Minibatch accuracy: 7.8%\n",
629-
"Validation accuracy: 10.4%\n",
630-
"Minibatch loss at step 10000 : 2.30177\n",
631-
"Minibatch accuracy: 10.2%\n",
632-
"Validation accuracy: 9.8%\n",
633-
"Minibatch loss at step 11000 : 2.30296\n",
634-
"Minibatch accuracy: 10.2%\n",
635-
"Validation accuracy: 10.4%\n",
636-
"Minibatch loss at step 12000 : 2.30295\n",
637-
"Minibatch accuracy: 10.9%\n",
638-
"Validation accuracy: 9.8%\n"
639-
]
640-
},
641-
{
642-
"ename": "KeyboardInterrupt",
643-
"evalue": "",
644-
"output_type": "error",
645-
"traceback": [
646-
"\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
647-
"\u001b[1;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
648-
"\u001b[1;32m<ipython-input-12-6e5cff3f9f17>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 73\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[0mtf_train_dataset\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mbatch_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtf_train_labels\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 74\u001b[0m _, l, predictions = session.run(\n\u001b[1;32m---> 75\u001b[1;33m [optimizer, loss, train_prediction], feed_dict=feed_dict)\n\u001b[0m\u001b[0;32m 76\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mstep\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m1000\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 77\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Minibatch loss at step\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\":\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
649-
"\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict)\u001b[0m\n\u001b[0;32m 383\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 384\u001b[0m \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 385\u001b[1;33m \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munique_fetch_targets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 386\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 387\u001b[0m \u001b[1;31m# User may have fetched the same tensor multiple times, but we\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
650-
"\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, target_list, fetch_list, feed_dict)\u001b[0m\n\u001b[0;32m 443\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 444\u001b[0m return tf_session.TF_Run(self._session, feed_dict, fetch_list,\n\u001b[1;32m--> 445\u001b[1;33m target_list)\n\u001b[0m\u001b[0;32m 446\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 447\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStatusNotOK\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
651-
"\u001b[1;31mKeyboardInterrupt\u001b[0m: "
603+
"Validation accuracy: 11.2%\n",
604+
"Minibatch loss at step 1000 : 2.46086\n",
605+
"Minibatch accuracy: 23.4%\n",
606+
"Validation accuracy: 37.1%\n",
607+
"Minibatch loss at step 2000 : 1.79268\n",
608+
"Minibatch accuracy: 35.9%\n",
609+
"Validation accuracy: 49.8%\n",
610+
"Minibatch loss at step 3000 : 1.76012\n",
611+
"Minibatch accuracy: 44.5%\n",
612+
"Validation accuracy: 56.3%\n",
613+
"Minibatch loss at step 4000 : 1.49048\n",
614+
"Minibatch accuracy: 50.0%\n",
615+
"Validation accuracy: 60.7%\n",
616+
"Minibatch loss at step 5000 : 1.35661\n",
617+
"Minibatch accuracy: 53.1%\n",
618+
"Validation accuracy: 64.0%\n",
619+
"Minibatch loss at step 6000 : 1.53\n",
620+
"Minibatch accuracy: 50.0%\n",
621+
"Validation accuracy: 66.7%\n",
622+
"Minibatch loss at step 7000 : 1.16328\n",
623+
"Minibatch accuracy: 63.3%\n",
624+
"Validation accuracy: 69.6%\n",
625+
"Minibatch loss at step 8000 : 1.34582\n",
626+
"Minibatch accuracy: 63.3%\n",
627+
"Validation accuracy: 70.1%\n",
628+
"Minibatch loss at step 9000 : 1.13307\n",
629+
"Minibatch accuracy: 60.2%\n",
630+
"Validation accuracy: 72.8%\n",
631+
"Minibatch loss at step 10000 : 1.1888\n",
632+
"Minibatch accuracy: 63.3%\n",
633+
"Validation accuracy: 73.5%\n",
634+
"Minibatch loss at step 11000 : 1.30667\n",
635+
"Minibatch accuracy: 54.7%\n",
636+
"Validation accuracy: 75.2%\n",
637+
"Minibatch loss at step 12000 : 1.19186\n",
638+
"Minibatch accuracy: 65.6%\n",
639+
"Validation accuracy: 76.1%\n",
640+
"Minibatch loss at step 13000 : 1.23864\n",
641+
"Minibatch accuracy: 64.8%\n",
642+
"Validation accuracy: 76.9%\n",
643+
"Minibatch loss at step 14000 : 1.00669\n",
644+
"Minibatch accuracy: 69.5%\n",
645+
"Validation accuracy: 77.0%\n",
646+
"Minibatch loss at step 15000 : 0.854794\n",
647+
"Minibatch accuracy: 72.7%\n",
648+
"Validation accuracy: 77.7%\n",
649+
"Minibatch loss at step 16000 : 0.820268\n",
650+
"Minibatch accuracy: 78.9%\n",
651+
"Validation accuracy: 78.0%\n",
652+
"Minibatch loss at step 17000 : 0.841344\n",
653+
"Minibatch accuracy: 74.2%\n",
654+
"Validation accuracy: 78.8%\n",
655+
"Minibatch loss at step 18000 : 0.783495\n",
656+
"Minibatch accuracy: 77.3%\n",
657+
"Validation accuracy: 78.8%\n",
658+
"Minibatch loss at step 19000 : 0.999198\n",
659+
"Minibatch accuracy: 71.1%\n",
660+
"Validation accuracy: 79.2%\n",
661+
"Minibatch loss at step 20000 : 0.834015\n",
662+
"Minibatch accuracy: 72.7%\n",
663+
"Validation accuracy: 79.7%\n",
664+
"Minibatch loss at step 21000 : 0.929157\n",
665+
"Minibatch accuracy: 71.9%\n",
666+
"Validation accuracy: 79.8%\n",
667+
"Minibatch loss at step 22000 : 0.705838\n",
668+
"Minibatch accuracy: 77.3%\n",
669+
"Validation accuracy: 80.4%\n",
670+
"Minibatch loss at step 23000 : 0.806978\n",
671+
"Minibatch accuracy: 73.4%\n",
672+
"Validation accuracy: 80.6%\n",
673+
"Minibatch loss at step 24000 : 0.700855\n",
674+
"Minibatch accuracy: 78.1%\n",
675+
"Validation accuracy: 80.8%\n",
676+
"Minibatch loss at step 25000 : 0.850926\n",
677+
"Minibatch accuracy: 76.6%\n",
678+
"Validation accuracy: 80.8%\n",
679+
"Minibatch loss at step 26000 : 0.92411\n",
680+
"Minibatch accuracy: 71.9%\n",
681+
"Validation accuracy: 81.1%\n",
682+
"Minibatch loss at step 27000 : 0.870534\n",
683+
"Minibatch accuracy: 73.4%\n",
684+
"Validation accuracy: 81.2%\n",
685+
"Minibatch loss at step 28000 : 0.655592\n",
686+
"Minibatch accuracy: 79.7%\n",
687+
"Validation accuracy: 81.5%\n",
688+
"Minibatch loss at step 29000 : 0.650673\n",
689+
"Minibatch accuracy: 79.7%\n",
690+
"Validation accuracy: 81.7%\n",
691+
"Minibatch loss at step 30000 : 0.575677\n",
692+
"Minibatch accuracy: 79.7%\n",
693+
"Validation accuracy: 81.9%\n",
694+
"Minibatch loss at step 31000 : 0.754482\n",
695+
"Minibatch accuracy: 77.3%\n",
696+
"Validation accuracy: 81.6%\n",
697+
"Minibatch loss at step 32000 : 0.647639\n",
698+
"Minibatch accuracy: 80.5%\n",
699+
"Validation accuracy: 82.2%\n",
700+
"Minibatch loss at step 33000 : 0.6582\n",
701+
"Minibatch accuracy: 80.5%\n",
702+
"Validation accuracy: 82.4%\n",
703+
"Minibatch loss at step 34000 : 0.517049\n",
704+
"Minibatch accuracy: 85.2%\n",
705+
"Validation accuracy: 82.3%\n",
706+
"Minibatch loss at step 35000 : 0.555218\n",
707+
"Minibatch accuracy: 83.6%\n",
708+
"Validation accuracy: 82.5%\n",
709+
"Minibatch loss at step 36000 : 0.674287\n",
710+
"Minibatch accuracy: 82.0%\n",
711+
"Validation accuracy: 82.6%\n",
712+
"Minibatch loss at step 37000 : 0.54334\n",
713+
"Minibatch accuracy: 85.2%\n",
714+
"Validation accuracy: 82.7%\n",
715+
"Minibatch loss at step 38000 : 0.626048\n",
716+
"Minibatch accuracy: 79.7%\n",
717+
"Validation accuracy: 82.9%\n",
718+
"Minibatch loss at step 39000 : 0.651072\n",
719+
"Minibatch accuracy: 77.3%\n",
720+
"Validation accuracy: 83.0%\n",
721+
"Minibatch loss at step 40000 : 0.625448\n",
722+
"Minibatch accuracy: 84.4%\n",
723+
"Validation accuracy: 83.2%\n",
724+
"Minibatch loss at step 41000 : 0.689314\n",
725+
"Minibatch accuracy: 77.3%\n",
726+
"Validation accuracy: 83.3%\n",
727+
"Minibatch loss at step 42000 : 0.657324\n",
728+
"Minibatch accuracy: 80.5%\n",
729+
"Validation accuracy: 83.3%\n",
730+
"Minibatch loss at step 43000 : 0.774549\n",
731+
"Minibatch accuracy: 75.0%\n",
732+
"Validation accuracy: 83.3%\n",
733+
"Minibatch loss at step 44000 : 0.592654\n",
734+
"Minibatch accuracy: 83.6%\n",
735+
"Validation accuracy: 83.7%\n",
736+
"Minibatch loss at step 45000 : 0.635808\n",
737+
"Minibatch accuracy: 78.1%\n",
738+
"Validation accuracy: 83.8%\n",
739+
"Minibatch loss at step 46000 : 0.401504\n",
740+
"Minibatch accuracy: 86.7%\n",
741+
"Validation accuracy: 83.6%\n",
742+
"Minibatch loss at step 47000 : 0.561845\n",
743+
"Minibatch accuracy: 81.2%\n",
744+
"Validation accuracy: 83.7%\n",
745+
"Minibatch loss at step 48000 : 0.55333\n",
746+
"Minibatch accuracy: 80.5%\n",
747+
"Validation accuracy: 83.8%\n",
748+
"Minibatch loss at step 49000 : 0.649688\n",
749+
"Minibatch accuracy: 81.2%\n",
750+
"Validation accuracy: 84.0%\n",
751+
"Minibatch loss at step 50000 : 0.565368\n",
752+
"Minibatch accuracy: 82.8%\n",
753+
"Validation accuracy: 84.0%\n",
754+
"Minibatch loss at step 51000 : 0.391173\n",
755+
"Minibatch accuracy: 89.1%\n",
756+
"Validation accuracy: 84.1%\n",
757+
"Minibatch loss at step 52000 : 0.459594\n",
758+
"Minibatch accuracy: 85.9%\n",
759+
"Validation accuracy: 84.1%\n",
760+
"Minibatch loss at step 53000 : 0.576485\n",
761+
"Minibatch accuracy: 78.1%\n",
762+
"Validation accuracy: 84.3%\n",
763+
"Minibatch loss at step 54000 : 0.572356\n",
764+
"Minibatch accuracy: 85.2%\n",
765+
"Validation accuracy: 84.3%\n",
766+
"Minibatch loss at step 55000 : 0.581976\n",
767+
"Minibatch accuracy: 82.8%\n",
768+
"Validation accuracy: 84.4%\n",
769+
"Minibatch loss at step 56000 : 0.526539\n",
770+
"Minibatch accuracy: 83.6%\n",
771+
"Validation accuracy: 84.5%\n",
772+
"Minibatch loss at step 57000 : 0.366597\n",
773+
"Minibatch accuracy: 88.3%\n",
774+
"Validation accuracy: 84.4%\n",
775+
"Minibatch loss at step 58000 : 0.413548\n",
776+
"Minibatch accuracy: 85.9%\n",
777+
"Validation accuracy: 84.5%\n",
778+
"Minibatch loss at step 59000 : 0.522779\n",
779+
"Minibatch accuracy: 84.4%\n",
780+
"Validation accuracy: 84.6%\n",
781+
"Minibatch loss at step 60000 : 0.474158\n",
782+
"Minibatch accuracy: 88.3%\n",
783+
"Validation accuracy: 84.7%\n",
784+
"Minibatch loss at step 61000 : 0.493542\n",
785+
"Minibatch accuracy: 85.9%\n",
786+
"Validation accuracy: 84.6%\n",
787+
"Minibatch loss at step 62000 : 0.655646\n",
788+
"Minibatch accuracy: 79.7%\n",
789+
"Validation accuracy: 84.8%\n",
790+
"Minibatch loss at step 63000 : 0.410069\n",
791+
"Minibatch accuracy: 85.9%\n",
792+
"Validation accuracy: 84.9%\n",
793+
"Minibatch loss at step 64000 : 0.577193\n",
794+
"Minibatch accuracy: 80.5%\n",
795+
"Validation accuracy: 84.9%\n",
796+
"Test accuracy: 91.6%\n"
652797
]
653798
}
654799
],
655800
"source": [
656801
"num_steps = 64001\n",
657802
"batch_size = 128\n",
658-
"hidden_size_1 = 1024\n",
659-
"hidden_size_2 = 300\n",
660-
"hidden_size_3 = 50\n",
803+
"hidden_size_1 = 256\n",
804+
"hidden_size_2 = 128\n",
805+
"hidden_size_3 = 64\n",
661806
"reg_term=0.001\n",
662807
"keep_prob=0.8\n",
663808
"\n",
@@ -683,17 +828,16 @@
683828
"\n",
684829
" # Training computation.\n",
685830
" def multi_layer_logit(input_dataset, keep_prob=keep_prob):\n",
686-
" hidden_1 = tf.nn.dropout(tf.nn.relu(tf.matmul(input_dataset, weights_h_1) + biases_h_1), keep_prob)\n",
687-
" hidden_2 = tf.nn.dropout(tf.nn.relu(tf.matmul(hidden_1, weights_h_2) + biases_h_2), keep_prob)\n",
688-
" hidden_3 = tf.nn.dropout(tf.nn.relu(tf.matmul(hidden_2, weights_h_3) + biases_h_3), keep_prob) \n",
831+
" hidden_1 = tf.nn.dropout(tf.nn.relu6(tf.matmul(input_dataset, weights_h_1) + biases_h_1), keep_prob)\n",
832+
" hidden_2 = tf.nn.dropout(tf.nn.relu6(tf.matmul(hidden_1, weights_h_2) + biases_h_2), keep_prob)\n",
833+
" hidden_3 = tf.nn.dropout(tf.nn.relu6(tf.matmul(hidden_2, weights_h_3) + biases_h_3), keep_prob) \n",
689834
"\n",
690835
" logits = tf.matmul(hidden_3, weights_y) + biases_y\n",
691836
" return logits\n",
692837
"\n",
693-
" logits = multi_layer_logit(tf_train_dataset, keep_prob=0.5)\n",
694-
" epsilon = 1e-9\n",
838+
" logits = multi_layer_logit(tf_train_dataset, keep_prob=keep_prob)\n",
695839
" loss = tf.reduce_mean(\n",
696-
" tf.nn.softmax_cross_entropy_with_logits(logits + epsilon, tf_train_labels))\n",
840+
" tf.nn.softmax_cross_entropy_with_logits(logits + 1e-12, tf_train_labels))\n",
697841
" \n",
698842
" loss_with_reg = loss + reg_term * (\n",
699843
" tf.nn.l2_loss(weights_h_1) +\n",
@@ -704,7 +848,7 @@
704848
"\n",
705849
" # Optimizer.\n",
706850
" global_step = tf.Variable(0)\n",
707-
" learning_rate = tf.train.exponential_decay(0.01, global_step, 10000, 0.95)\n",
851+
" learning_rate = tf.train.exponential_decay(0.05, global_step, 1000, 0.98)\n",
708852
" optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_with_reg, global_step=global_step)\n",
709853
"\n",
710854
" # Predictions for the training, validation, and test data.\n",

0 commit comments

Comments
 (0)