update

doongz · Feb 5, 2023 · 54f3287 · 54f3287
1 parent d23101b
commit 54f3287
Show file tree

Hide file tree

Showing 48 changed files with 60 additions and 14 deletions.
diff --git a/Lecture-1 Introduction of Deep Learning/1-regression (v16).pdf b/Lecture-1 Introduction of Deep Learning/1-regression (v16).pdf
diff --git a/Lecture-1/Colab Tutorial 2022.pdf → ... of Deep Learning/Colab Tutorial 2022.pdf b/Lecture-1/Colab Tutorial 2022.pdf → ... of Deep Learning/Colab Tutorial 2022.pdf
diff --git a/Lecture-1/EnvironmentSetup.pdf → ...ion of Deep Learning/EnvironmentSetup.pdf b/Lecture-1/EnvironmentSetup.pdf → ...ion of Deep Learning/EnvironmentSetup.pdf
diff --git a/Lecture-1/Pytorch Tutorial 1.pdf → ...n of Deep Learning/Pytorch Tutorial 1.pdf b/Lecture-1/Pytorch Tutorial 1.pdf → ...n of Deep Learning/Pytorch Tutorial 1.pdf
diff --git a/Lecture-1/Pytorch Tutorial 2.pdf → ...n of Deep Learning/Pytorch Tutorial 2.pdf b/Lecture-1/Pytorch Tutorial 2.pdf → ...n of Deep Learning/Pytorch Tutorial 2.pdf
diff --git a/Lecture-1/introduction (v2).pdf → ...on of Deep Learning/introduction (v2).pdf b/Lecture-1/introduction (v2).pdf → ...on of Deep Learning/introduction (v2).pdf
diff --git a/Lecture-1/rule (v2).pdf → ...troduction of Deep Learning/rule (v2).pdf b/Lecture-1/rule (v2).pdf → ...troduction of Deep Learning/rule (v2).pdf
diff --git a/Lecture-10/attack_v3.pdf → Lecture-10 Attack/attack_v3.pdf b/Lecture-10/attack_v3.pdf → Lecture-10 Attack/attack_v3.pdf
diff --git a/Lecture-11/More self-supervised (v2).pdf → ... Adaptation/More self-supervised (v2).pdf b/Lecture-11/More self-supervised (v2).pdf → ... Adaptation/More self-supervised (v2).pdf
diff --git a/Lecture-11/da_v6.pdf → Lecture-11 Adaptation/da_v6.pdf b/Lecture-11/da_v6.pdf → Lecture-11 Adaptation/da_v6.pdf
diff --git a/Lecture-12/drl_v5.pdf → Lecture-12 Reinforcement Learning/drl_v5.pdf b/Lecture-12/drl_v5.pdf → Lecture-12 Reinforcement Learning/drl_v5.pdf
diff --git a/Lecture-13/tiny_v7.pdf → Lecture-13 Network Compression/tiny_v7.pdf b/Lecture-13/tiny_v7.pdf → Lecture-13 Network Compression/tiny_v7.pdf
diff --git a/Lecture-14/life_v2.pdf → Lecture-14 Life-long Learning/life_v2.pdf b/Lecture-14/life_v2.pdf → Lecture-14 Life-long Learning/life_v2.pdf
diff --git a/Lecture-15/More Meta (v6).pdf → Lecture-15 Meta Learning/More Meta (v6).pdf b/Lecture-15/More Meta (v6).pdf → Lecture-15 Meta Learning/More Meta (v6).pdf
diff --git a/Lecture-2 What to do if my network fails to train/1-overfit-v6.pdf b/Lecture-2 What to do if my network fails to train/1-overfit-v6.pdf
diff --git a/Lecture-2 What to do if my network fails to train/2-small-gradient-v7.pdf b/Lecture-2 What to do if my network fails to train/2-small-gradient-v7.pdf
diff --git a/Lecture-2 What to do if my network fails to train/3-optimizer_v4.pdf b/Lecture-2 What to do if my network fails to train/3-optimizer_v4.pdf
diff --git a/Lecture-2 What to do if my network fails to train/4-classification_v2.pdf b/Lecture-2 What to do if my network fails to train/4-classification_v2.pdf
diff --git a/Lecture-2/theory (v7).pdf → ... network fails to train/5-theory (v7).pdf b/Lecture-2/theory (v7).pdf → ... network fails to train/5-theory (v7).pdf
diff --git a/Lecture-2/validation.pdf → ...y network fails to train/6-validation.pdf b/Lecture-2/validation.pdf → ...y network fails to train/6-validation.pdf
diff --git a/Lecture-2 What to do if my network fails to train/7-Optimization.pdf b/Lecture-2 What to do if my network fails to train/7-Optimization.pdf
diff --git a/Lecture-3 Image as input/1-cnn_v4.pdf b/Lecture-3 Image as input/1-cnn_v4.pdf
diff --git a/Lecture-3/whydeep (v4).pdf → Lecture-3 Image as input/2-whydeep (v4).pdf b/Lecture-3/whydeep (v4).pdf → Lecture-3 Image as input/2-whydeep (v4).pdf
diff --git a/Lecture-4 Sequence as input/1-self_v7.pdf b/Lecture-4 Sequence as input/1-self_v7.pdf
diff --git a/Lecture-4 Sequence as input/2-GNN.pdf b/Lecture-4 Sequence as input/2-GNN.pdf
diff --git a/Lecture-5 Sequence to sequence/1-normalization_v4.pdf b/Lecture-5 Sequence to sequence/1-normalization_v4.pdf
diff --git a/Lecture-5 Sequence to sequence/2-seq2seq_v9.pdf b/Lecture-5 Sequence to sequence/2-seq2seq_v9.pdf
diff --git a/Lecture-5/xformer (v8).pdf → ...5 Sequence to sequence/3-xformer (v8).pdf b/Lecture-5/xformer (v8).pdf → ...5 Sequence to sequence/3-xformer (v8).pdf
diff --git a/Lecture-6/gan_v10.pdf → Lecture-6 Generation/gan_v10.pdf b/Lecture-6/gan_v10.pdf → Lecture-6 Generation/gan_v10.pdf
diff --git a/Self-supervised learning for NLP/PLM.pdf → ... Self-supervised learning for NLP/PLM.pdf b/Self-supervised learning for NLP/PLM.pdf → ... Self-supervised learning for NLP/PLM.pdf
diff --git a/Self-supervised learning for NLP/bert_v8.pdf → ...f-supervised learning for NLP/bert_v8.pdf b/Self-supervised learning for NLP/bert_v8.pdf → ...f-supervised learning for NLP/bert_v8.pdf
diff --git a/Lecture-7/SSL_speech_image (v9).pdf → ...peech and Image/SSL_speech_image (v9).pdf b/Lecture-7/SSL_speech_image (v9).pdf → ...peech and Image/SSL_speech_image (v9).pdf
diff --git a/Lecture-8/auto_v8.pdf → ...uto-encoder Anomaly Detection/auto_v8.pdf b/Lecture-8/auto_v8.pdf → ...uto-encoder Anomaly Detection/auto_v8.pdf
diff --git a/Lecture-9/xai_v4.pdf → Lecture-9 Explainable AI/xai_v4.pdf b/Lecture-9/xai_v4.pdf → Lecture-9 Explainable AI/xai_v4.pdf
diff --git a/README.md b/README.md
@@ -45,6 +45,16 @@
 
 ### Lecture 3: Image as input
 
+cnn
+
+<img src="./images/image-20230205125441972.png" alt="image-20230205125441972" style="zoom:50%;" />
+
+<img src="./images/image-20230205131700429.png" alt="image-20230205131700429" style="zoom:50%;" />
+
+<img src="./images/image-20230205132102613.png" alt="image-20230205132102613" style="zoom:50%;" />
+
+<img src="./images/image-20230205132131344.png" alt="image-20230205132131344" style="zoom:50%;" />
+
 深度学习好在哪
 
 ReLU 进行叠加 + 常数可代表任何方程
@@ -57,32 +67,68 @@ ReLU 进行叠加 + 常数可代表任何方程
 
 <img src="./images/image-20230108232105650.png" alt="image-20230108232105650" style="zoom:50%;" />
 
-Lecture 4: Sequence as input
+### Lecture 4: Sequence as input
+
+<img src="./images/image-20230205135807348.png" alt="image-20230205135807348" style="zoom:50%;" />
+
+<img src="./images/image-20230205142626232.png" alt="image-20230205142626232" style="zoom:50%;" />
+
+step-1 从 a_i 得到 q k v 的矩阵
+
+<img src="./images/image-20230205143344853.png" alt="image-20230205143344853" style="zoom:50%;" />
+
+step-2 K 和 Q 的矩阵乘 得到 A 矩阵
+
+<img src="./images/image-20230205143822509.png" alt="image-20230205143822509" style="zoom:50%;" />
+
+step-3 V 和 A 的矩阵乘得到 O 矩阵
+
+<img src="./images/image-20230205144116182.png" alt="image-20230205144116182" style="zoom:50%;" />
+
+并发能力由硬件提供，整个过程需要训练的是 q k v 的 weight 矩阵
+
+<img src="./images/image-20230205144211354.png" alt="image-20230205144211354" style="zoom:50%;" />
+
+<img src="./images/image-20230205151409655.png " alt="image-20230205151409655" style="zoom:50%;" />
+
+<img src="./images/image-20230205151633597.png" alt="image-20230205151633597" style="zoom:50%;" />
+
+self-attention vs CNN
+
+- CNN 是 self-attention 的特例，CNN 的 receptive field 是固定的，self-attention 的 receptive field 是可变的，甚至是全图，甚至是可以训练的
+- 训练集小的时候 CNN 效果好些，在大训练集上 self-attention 效果更好
+
+<img src="./images/image-20230205152152888.png" alt="image-20230205152152888" style="zoom:50%;" />
+
+self-attention vs RNN
+
+- RNN 中的输入是有距离的（第一个输入在最后面的计算时慢慢会被遗忘），self-attention 中所有的输入都是平等的
+- RNN 只能一个一个去算，self-attention 可以并行计算
+
+### Lecture 5: Sequence to sequence
 
 <img src="./images/image-20230115143207868.png" alt="image-20230115143207868" style="zoom: 33%;" />
 
 <img src="./images/image-20230115143556724.png" alt="image-20230115143556724" style="zoom:33%;" />
 
-Lecture 5: Sequence to sequence
-
-Lecture 6: Generation
+### Lecture 6: Generation
 
-Recent Advance of Self-supervised learning for NLP
+### Recent Advance of Self-supervised learning for NLP
 
-Lecture 7: Self-supervised learning for Speech and Image
+### Lecture 7: Self-supervised learning for Speech and Image
 
-Lecture 8: Auto-encoder/ Anomaly Detection
+### Lecture 8: Auto-encoder/ Anomaly Detection
 
-Lecture 9: Explainable AI
+### Lecture 9: Explainable AI
 
-Lecture 10: Attack
+### Lecture 10: Attack
 
-Lecture 11: Adaptation
+### Lecture 11: Adaptation
 
-Lecture 12: Reinforcement Learning
+### Lecture 12: Reinforcement Learning
 
-Lecture 13: Network Compression
+### Lecture 13: Network Compression
 
-Lecture 14: Life-long Learning
+### Lecture 14: Life-long Learning
 
-Lecture 15: Meta Learning
+### Lecture 15: Meta Learning
diff --git a/images/image-20230205125441972.png b/images/image-20230205125441972.png
diff --git a/images/image-20230205131700429.png b/images/image-20230205131700429.png
diff --git a/images/image-20230205132102613.png b/images/image-20230205132102613.png
diff --git a/images/image-20230205132131344.png b/images/image-20230205132131344.png
diff --git a/images/image-20230205135807348.png b/images/image-20230205135807348.png
diff --git a/images/image-20230205142626232.png b/images/image-20230205142626232.png
diff --git a/images/image-20230205143344853.png b/images/image-20230205143344853.png
diff --git a/images/image-20230205143822509.png b/images/image-20230205143822509.png
diff --git a/images/image-20230205144116182.png b/images/image-20230205144116182.png
diff --git a/images/image-20230205144211354.png b/images/image-20230205144211354.png
diff --git a/images/image-20230205151409655.png b/images/image-20230205151409655.png
diff --git a/images/image-20230205151633597.png b/images/image-20230205151633597.png
diff --git a/images/image-20230205152152888.png b/images/image-20230205152152888.png