-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathindex.html
864 lines (518 loc) · 548 KB
/
index.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>DoubleHappy or Jepson</title>
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta property="og:type" content="website">
<meta property="og:title" content="DoubleHappy or Jepson">
<meta property="og:url" content="http://yoursite.com/index.html">
<meta property="og:site_name" content="DoubleHappy or Jepson">
<meta property="og:locale" content="en">
<meta name="twitter:card" content="summary">
<link rel="alternate" href="/atom.xml" title="DoubleHappy or Jepson" type="application/atom+xml">
<link rel="icon" href="/images/default-avatar.jpg">
<link href="//fonts.googleapis.com/css?family=Source+Code+Pro" rel="stylesheet" type="text/css">
<link rel="stylesheet" href="/css/style.css">
<link rel="stylesheet" href="/css/highlight.css">
</head>
<body>
<div id="fullpage" class="mobile-nav-right">
<div id="wrapper" title="图片来自网络">
<header id="header">
<div id="nav-toggle" class="nav-toggle"></div>
<div class="head-box global-width">
<nav class="nav-box nav-right">
<a class="nav-item" href="/archives" title
>首页</a>
<a class="nav-item" href="/archives" title
>归档</a>
</nav>
</div>
</header>
<div id="middlecontent" title class="global-width sidebar-right">
<section id="main">
<article id="post-docker-doublehappy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/04/27/docker-doublehappy/">docker--doublehappy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/04/27/docker-doublehappy/" class="article-date">
<time datetime="2020-04-27T09:27:30.000Z" itemprop="datePublished">2020-04-27</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<p><a href="https://docs.docker.com/" target="_blank" rel="noopener">docker文档官网</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">docker 一定是选用 社区版本的 花钱的版本我们不用的</span><br></pre></td></tr></table></figure></div>
<p><a href="https://docs.docker.com/install/linux/docker-ce/centos/" target="_blank" rel="noopener">docker-linux-centos</a></p>
<h2 id="部署"><a href="#部署" class="headerlink" title="部署"></a>部署</h2><p><img src="https://img-blog.csdnimg.cn/20200206140123182.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">按照官网走即可:</span><br><span class="line"></span><br><span class="line"><span class="number">1</span>.部署</span><br><span class="line"></span><br><span class="line">安装 yum工具：</span><br><span class="line">yum install -y yum-utils \</span><br><span class="line">device-mapper-persistent-data \</span><br><span class="line">lvm2</span><br><span class="line">----------------------------------------------</span><br><span class="line">安装dockeryum源：</span><br><span class="line"></span><br><span class="line">yum-config-manager \</span><br><span class="line">--add-repo \</span><br><span class="line">https://download.docker.com/linux/centos/docker-ce.repo</span><br><span class="line">----------------------------------------------</span><br><span class="line">安装docker的引擎：</span><br><span class="line"></span><br><span class="line">yum install -y docker-ce docker-ce-cli containerd.io</span><br><span class="line">----------------------------------------------</span><br><span class="line">启动docker：</span><br><span class="line"></span><br><span class="line">systemctl start docker</span><br><span class="line">systemctl status docker</span><br><span class="line"></span><br><span class="line">----------------------------------------------</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> 上面的第二步骤：</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># yum-config-manager \</span></span><br><span class="line">> --add-repo \</span><br><span class="line">> https://download.docker.com/linux/centos/docker-ce.repo</span><br><span class="line">Loaded plugins: fastestmirror</span><br><span class="line">adding repo from: https://download.docker.com/linux/centos/docker-ce.repo</span><br><span class="line">grabbing file https://download.docker.com/linux/centos/docker-ce.repo to /etc/yum.repos.d/docker-ce.repo</span><br><span class="line">repo saved to /etc/yum.repos.d/docker-ce.repo</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># ll /etc/yum.repos.d</span></span><br><span class="line">total <span class="number">16</span></span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">675</span> Sep <span class="number">11</span> <span class="number">09</span>:<span class="number">44</span> CentOS-Base.repo</span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">2424</span> Oct <span class="number">19</span> <span class="number">05</span>:<span class="number">57</span> docker-ce.repo</span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">230</span> Sep <span class="number">11</span> <span class="number">09</span>:<span class="number">44</span> epel.repo</span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">113</span> Jul <span class="number">15</span> <span class="number">2014</span> nginx.repo</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">/etc/yum.repos.d：</span><br><span class="line"> 这个目录就是你 yum 源所安装的东西 </span><br><span class="line"></span><br><span class="line">可以看一下：</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># cat /etc/yum.repos.d/docker-ce.repo</span></span><br><span class="line">[docker-ce-stable]</span><br><span class="line">name=Docker CE Stable - <span class="variable">$basearch</span></span><br><span class="line">baseurl=https://download.docker.com/linux/centos/<span class="number">7</span>/<span class="variable">$basearch</span>/stable</span><br><span class="line">enabled=<span class="number">1</span></span><br><span class="line">gpgcheck=<span class="number">1</span></span><br><span class="line">gpgkey=https://download.docker.com/linux/centos/gpg</span><br><span class="line"></span><br><span class="line">[docker-ce-stable-debuginfo]</span><br><span class="line">name=Docker CE Stable - Debuginfo <span class="variable">$basearch</span></span><br><span class="line">baseurl=https://download.docker.com/linux/centos/<span class="number">7</span>/debug-<span class="variable">$basearch</span>/stable</span><br><span class="line">enabled=</span><br><span class="line">gpgcheck=<span class="number">1</span></span><br><span class="line">gpgkey=https://download.docker.com/linux/centos/gpg</span><br><span class="line">......</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">这就是 yum源 到时候去这些地址 下载东西</span><br></pre></td></tr></table></figure></div>
<h2 id="docker-hub"><a href="#docker-hub" class="headerlink" title="docker hub"></a>docker hub</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">docker官方的镜像仓库地址: 与github是类似的 </span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"><span class="number">1</span>.docker hub 就是存放我们的镜像的</span><br><span class="line"><span class="number">2</span>.image 镜像</span><br><span class="line"><span class="number">3</span>.container 镜像文件创建的实例</span><br><span class="line"></span><br><span class="line">eg：</span><br><span class="line"> win7 系统盘 iso镜像文件 ==》VM工具==》win7-<span class="number">1</span> win7-<span class="number">2</span> 实例</span><br><span class="line">即：</span><br><span class="line"> win7 系统盘 iso镜像文件 就是 image</span><br><span class="line"> win7-<span class="number">1</span> win7-<span class="number">2</span> 实例 就是 container</span><br></pre></td></tr></table></figure></div>
<p><a href="https://www.docker.com/products/docker-hub" target="_blank" rel="noopener">docker官方的镜像仓库地址</a></p>
<h2 id="案例"><a href="#案例" class="headerlink" title="案例"></a>案例</h2><p><strong>hello world</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br></pre></td><td class="code"><pre><span class="line">docker help </span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker search hello</span></span><br><span class="line">NAME DESCRIPTION STARS OFFICIAL AUTOMATED</span><br><span class="line">hello-world Hello World! (an example of minimal Dockeriz… <span class="number">1121</span> [OK] </span><br><span class="line">kitematic/hello-world-nginx A light-weight nginx container that demonstr… <span class="number">138</span> </span><br><span class="line"></span><br><span class="line">我们找一个案例 hello-world 官网的</span><br><span class="line"></span><br><span class="line"><span class="number">1</span>.先把 需要的 pull 下来 </span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker pull hello-world</span></span><br><span class="line">Using default tag: latest</span><br><span class="line">latest: Pulling from library/hello-world</span><br><span class="line"><span class="number">1</span>b930d010525: Pull complete </span><br><span class="line">Digest: sha256:<span class="number">9572</span>f7cdcee8591948c2963463447a53466950b3fc15a247fcad1917ca215a2f</span><br><span class="line">Status: Downloaded newer image <span class="keyword">for</span> hello-world:latest</span><br><span class="line">docker.io/library/hello-world:latest</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"> hello-world:latest</span><br><span class="line">指的是：</span><br><span class="line"> 镜像的名字 ： 镜像的版本</span><br><span class="line"></span><br><span class="line"><span class="number">2</span>.看一下我们机器上有那些镜像</span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker images</span></span><br><span class="line">REPOSITORY TAG IMAGE ID CREATED SIZE</span><br><span class="line">apache/kudu latest e1c6e5b62086 <span class="number">2</span> months ago <span class="number">476</span>MB</span><br><span class="line">hello-world latest fce289e99eb9 <span class="number">13</span> months ago <span class="number">1.84</span>kB</span><br><span class="line">kunickiaj/kudu latest c30c95f5fe58 <span class="number">17</span> months ago <span class="number">896</span>MB</span><br><span class="line">docker/compose <span class="number">1.8</span>.<span class="number">0</span> <span class="number">89188432</span>ef03 <span class="number">3</span> years ago <span class="number">59.1</span>MB</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"><span class="number">3</span>. 通过 镜像模版 生成一个 容器（实例）</span><br><span class="line"> 通过 image 生成一个 container</span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker run hello-world</span></span><br><span class="line"></span><br><span class="line">Hello from Docker!</span><br><span class="line">This message shows that your installation appears to be working correctly.</span><br><span class="line"></span><br><span class="line">To generate this message, Docker took the following steps:</span><br><span class="line"> <span class="number">1</span>. The Docker client contacted the Docker daemon.</span><br><span class="line"> <span class="number">2</span>. The Docker daemon pulled the <span class="string">"hello-world"</span> image from the Docker Hub.</span><br><span class="line"> (amd64)</span><br><span class="line"> <span class="number">3</span>. The Docker daemon created a new container from that image which runs the</span><br><span class="line"> executable that produces the output you are currently reading.</span><br><span class="line"> <span class="number">4</span>. The Docker daemon streamed that output to the Docker client, which sent it</span><br><span class="line"> to your terminal.</span><br><span class="line"></span><br><span class="line">To <span class="keyword">try</span> something more ambitious, you can run an Ubuntu container with:</span><br><span class="line"> $ docker run -it ubuntu bash</span><br><span class="line"></span><br><span class="line">Share images, automate workflows, and more with a free Docker ID:</span><br><span class="line"> https://hub.docker.com/</span><br><span class="line"></span><br><span class="line"><span class="keyword">For</span> more examples and ideas, visit:</span><br><span class="line"> https://docs.docker.com/get-started/</span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"><span class="number">4</span>.查看运行生成的实例</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker ps -a</span></span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">a38e8df475a1 hello-world <span class="string">"/hello"</span> About a minute ago Exited (<span class="number">0</span>) About a minute ago strange_thompson</span><br><span class="line">cc897cbba1ae kunickiaj/kudu <span class="string">"/docker-entrypoint.…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago kudu-tserver</span><br><span class="line"><span class="number">5402</span>fe1c9d3a kunickiaj/kudu <span class="string">"/docker-entrypoint.…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago kudu-master</span><br><span class="line">c36ac4effd05 kunickiaj/kudu <span class="string">"/docker-entrypoint.…"</span> <span class="number">8</span> weeks ago Created kudu-tserver-data</span><br><span class="line"><span class="number">37</span>f0ad272538 kunickiaj/kudu <span class="string">"/docker-entrypoint.…"</span> <span class="number">8</span> weeks ago Created kudu-master-data</span><br><span class="line">ee07f7ee2717 apache/kudu:latest <span class="string">"/kudu-entrypoint.sh…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago docker_kudu-tserver_1</span><br><span class="line"><span class="number">1</span>a1ae66602ad apache/kudu:latest <span class="string">"/kudu-entrypoint.sh…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago docker_kudu-master-<span class="number">1</span>_1</span><br><span class="line">f440472b35b2 apache/kudu:latest <span class="string">"/kudu-entrypoint.sh…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago docker_kudu-master-<span class="number">2</span>_1</span><br><span class="line"><span class="number">1</span>b32425575bb apache/kudu:latest <span class="string">"/kudu-entrypoint.sh…"</span> <span class="number">8</span> weeks ago Exited (<span class="number">137</span>) <span class="number">6</span> weeks ago docker_kudu-master-<span class="number">3</span>_1</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">第三波的打印：就是 docker的步骤</span><br><span class="line"></span><br><span class="line"><span class="number">1</span>.docker 有 客户端 + 服务端 </span><br><span class="line"> 即：docker命令 就是 client</span><br><span class="line"> 后台 status 显示的进程 就是 daemon（服务段）</span><br><span class="line"><span class="number">2</span>.client 仅仅 用来 发送指令的 </span><br><span class="line"> 其他的 run 等 全是 服务段来做的</span><br></pre></td></tr></table></figure></div>
<p><strong>nginx</strong></p>
<p><img src="https://img-blog.csdnimg.cn/20200206144443384.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">去 docker hub 上面 搜一下 使用官方的</span><br></pre></td></tr></table></figure></div>
<p><a href="https://hub.docker.com/_/nginx?tab=tags" target="_blank" rel="noopener">nginx的dock！！！</a><br><img src="https://img-blog.csdnimg.cn/20200206144710722.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<p><img src="https://img-blog.csdnimg.cn/20200206144958604.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre></td><td class="code"><pre><span class="line">在上面的url 的 描述选项 可以知道如何使用 nginx：</span><br><span class="line"></span><br><span class="line"><span class="number">1</span>.docker pull nginx:latest</span><br><span class="line"></span><br><span class="line"><span class="number">2</span>.docker run \</span><br><span class="line">--name doublehappy-nginx1 \</span><br><span class="line">-d -p <span class="number">8080</span>:<span class="number">80</span> \</span><br><span class="line">nginx:latest</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">--name 指定 镜像实例的名字</span><br><span class="line">nginx:latest 指的是 你来自哪个镜像的 </span><br><span class="line"><span class="number">8080</span>:<span class="number">80</span> </span><br><span class="line"> <span class="number">8080</span> 指linux的端口 <span class="number">80</span> 指的是 容器的端口 即 我访问linux8080 会映射到docker <span class="number">80</span> 端口</span><br><span class="line"> </span><br><span class="line"><span class="number">3</span>. 查看现在存活的 </span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker ps </span></span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">d4267c461c6c nginx:latest <span class="string">"nginx -g 'daemon of…"</span> <span class="number">40</span> seconds ago Up <span class="number">40</span> seconds <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8080</span>-><span class="number">80</span>/tcp doublehappy-nginx1</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">加上 -a 连推出过的 都看</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8080</span>-><span class="number">80</span>/tcp </span><br><span class="line">试着 页面访问 通过 linux <span class="number">8080</span></span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200206155918366.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">说明：</span><br><span class="line"> 我们通过docker 启动的一个 实例 </span><br><span class="line"> 容器里 运行着nginx 是可以访问的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">4</span>.进入 nginx 的 </span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker exec -it d4267c461c6c /bin/bash</span></span><br><span class="line">root@d4267c461c6c:/<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">看一个东西：</span><br><span class="line">root@d4267c461c6c:/<span class="comment"># ls -l /</span></span><br><span class="line">total <span class="number">64</span></span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> bin</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Nov <span class="number">10</span> <span class="number">12</span>:<span class="number">17</span> boot</span><br><span class="line">drwxr-xr-x <span class="number">5</span> root root <span class="number">340</span> Feb <span class="number">6</span> <span class="number">07</span>:<span class="number">49</span> dev</span><br><span class="line">drwxr-xr-x <span class="number">1</span> root root <span class="number">4096</span> Feb <span class="number">6</span> <span class="number">07</span>:<span class="number">49</span> etc</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Nov <span class="number">10</span> <span class="number">12</span>:<span class="number">17</span> home</span><br><span class="line">drwxr-xr-x <span class="number">1</span> root root <span class="number">4096</span> Feb <span class="number">2</span> <span class="number">08</span>:<span class="number">06</span> lib</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> lib64</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> media</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> mnt</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> opt</span><br><span class="line">dr-xr-xr-x <span class="number">93</span> root root <span class="number">0</span> Feb <span class="number">6</span> <span class="number">07</span>:<span class="number">49</span> proc</span><br><span class="line">drwx------ <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> root</span><br><span class="line">drwxr-xr-x <span class="number">1</span> root root <span class="number">4096</span> Feb <span class="number">6</span> <span class="number">07</span>:<span class="number">49</span> run</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> sbin</span><br><span class="line">drwxr-xr-x <span class="number">2</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> srv</span><br><span class="line">dr-xr-xr-x <span class="number">13</span> root root <span class="number">0</span> Feb <span class="number">6</span> <span class="number">07</span>:<span class="number">49</span> sys</span><br><span class="line">drwxrwxrwt <span class="number">1</span> root root <span class="number">4096</span> Feb <span class="number">2</span> <span class="number">08</span>:<span class="number">06</span> tmp</span><br><span class="line">drwxr-xr-x <span class="number">1</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> usr</span><br><span class="line">drwxr-xr-x <span class="number">1</span> root root <span class="number">4096</span> Jan <span class="number">30</span> <span class="number">00</span>:<span class="number">00</span> var</span><br><span class="line">root@d4267c461c6c:/<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">这就是容器里的 结构</span><br><span class="line"></span><br><span class="line">下面画个图理解</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200206160811416.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">5</span>.nginx 映射一个 别的页面</span><br><span class="line"></span><br><span class="line">emmm 准备一个html： 给他映射到 我git博客上</span><br><span class="line">-------------------------------</span><br><span class="line"><!DOCTYPE html></span><br><span class="line"><html></span><br><span class="line"><head></span><br><span class="line"><title>Welcome to nginx!</title></span><br><span class="line"><style></span><br><span class="line"> body {</span><br><span class="line"> width: <span class="number">35</span>em;</span><br><span class="line"> margin: <span class="number">0</span> auto;</span><br><span class="line"> font-family: Tahoma, Verdana, Arial, sans-serif;</span><br><span class="line"> }</span><br><span class="line"></style></span><br><span class="line"></head></span><br><span class="line"><body></span><br><span class="line"></span><br><span class="line"><p id=<span class="string">"p1"</span>>Hello Funs</p></span><br><span class="line"><a href=<span class="string">"https://sxwanggit126.github.io/archives/"</span>>my blog</a></span><br><span class="line"></body></span><br><span class="line"></html></span><br><span class="line"></span><br><span class="line">准备：</span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># vim doublehappy.html</span></span><br><span class="line"><!DOCTYPE html></span><br><span class="line"><html></span><br><span class="line"> ead></span><br><span class="line">▽title>Welcome to nginx!</title></span><br><span class="line"><style></span><br><span class="line"> body {</span><br><span class="line"> width: <span class="number">35</span>em;</span><br><span class="line"> margin: <span class="number">0</span> auto;</span><br><span class="line"> font-family: Tahoma, Verdana, Arial, sans-serif;</span><br><span class="line"> }</span><br><span class="line"></style></span><br><span class="line"></head></span><br><span class="line"><body></span><br><span class="line"></span><br><span class="line"><p id=<span class="string">"p1"</span>>Hello Funs</p></span><br><span class="line"><a href=<span class="string">"https://sxwanggit126.github.io/archives/"</span>>www.ruozedata.com</a></span><br><span class="line"></body></span><br><span class="line"></html></span><br><span class="line">~ </span><br><span class="line">~ </span><br><span class="line"><span class="string">"doublehappy.html"</span> [New] <span class="number">18</span>L, <span class="number">330</span>C written </span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># vim doublehappy.html</span></span><br><span class="line"><!DOCTYPE html></span><br><span class="line"><html></span><br><span class="line"><head></span><br><span class="line"><title>Welcome to nginx!</title></span><br><span class="line"><style></span><br><span class="line"> body {</span><br><span class="line">▽ width: <span class="number">35</span>em;</span><br><span class="line"> margin: <span class="number">0</span> auto;</span><br><span class="line"> font-family: Tahoma, Verdana, Arial, sans-serif;</span><br><span class="line"> }</span><br><span class="line"></style></span><br><span class="line"></head></span><br><span class="line"><body></span><br><span class="line"></span><br><span class="line"><p id=<span class="string">"p1"</span>>Hello Funs</p></span><br><span class="line"><a href=<span class="string">"https://sxwanggit126.github.io/archives/"</span>>www.ruozedata.com</a></span><br><span class="line"></body></span><br><span class="line"></html></span><br><span class="line">~ </span><br><span class="line">~ </span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># ll</span></span><br><span class="line">total <span class="number">4</span></span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">330</span> Feb <span class="number">6</span> <span class="number">16</span>:<span class="number">16</span> doublehappy.html</span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># mv doublehappy.html index.html</span></span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># vim index.html </span></span><br><span class="line"><!DOCTYPE html></span><br><span class="line"><html></span><br><span class="line"><head></span><br><span class="line"><title>Welcome to nginx!</title></span><br><span class="line"><style></span><br><span class="line"> body {</span><br><span class="line"> width: <span class="number">35</span>em;</span><br><span class="line"> margin: <span class="number">0</span> auto;</span><br><span class="line"> font-family: Tahoma, Verdana, Arial, sans-serif;</span><br><span class="line"> }</span><br><span class="line"></style></span><br><span class="line"></head></span><br><span class="line"><body></span><br><span class="line"></span><br><span class="line"><p id=<span class="string">"p1"</span>>Hello Funs</p></span><br><span class="line"><a href=<span class="string">"https://sxwanggit126.github.io/archives/"</span>>www.ruozedata.com</a></span><br><span class="line"></body></span><br><span class="line"></html></span><br><span class="line">~ </span><br><span class="line">~ </span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># pwd</span></span><br><span class="line">/root/nginx</span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">默认 是 index.xml</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">命令：我们换一个 nginx 同时 换一个 端口号</span><br><span class="line"></span><br><span class="line">docker run \</span><br><span class="line">--name doublehappy-nginx2 \</span><br><span class="line">-v /root/nginx:/usr/share/nginx/html:ro \</span><br><span class="line">-d -p <span class="number">8081</span>:<span class="number">80</span> \</span><br><span class="line">nginx:latest</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">-v：指的是 将本地的文件夹映射到 容器里面的文件夹</span><br><span class="line">ro：指的是 你挂载的文件 是否可读和可写 r可读 w可写 即：rw</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># docker ps</span></span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line"><span class="number">7</span>fbe515cc1ee nginx:latest <span class="string">"nginx -g 'daemon of…"</span> <span class="number">7</span> seconds ago Up <span class="number">6</span> seconds <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8081</span>-><span class="number">80</span>/tcp doublehappy-nginx2</span><br><span class="line">d4267c461c6c nginx:latest <span class="string">"nginx -g 'daemon of…"</span> <span class="number">30</span> minutes ago Up <span class="number">30</span> minutes <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8080</span>-><span class="number">80</span>/tcp doublehappy-nginx1</span><br><span class="line">[root@hadoop101 nginx]<span class="comment">#</span></span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200206162332705.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br>点击链接 即可 进入映射的url<br><img src="https://img-blog.csdnimg.cn/20200206162410539.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">进入 容器：</span><br><span class="line">[root@hadoop101 nginx]<span class="comment"># docker exec -it doublehappy-nginx2 /bin/bash</span></span><br><span class="line">root@<span class="number">7</span>fbe515cc1ee:/<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">/bin/bash 指的是 以什么进行 解析 </span><br><span class="line">因为 这个容器 相当于 运行一个 小的 linux 以 bash来进行解析</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">root@<span class="number">7</span>fbe515cc1ee:/<span class="comment"># ls -l /usr/share/nginx/html</span></span><br><span class="line">total <span class="number">4</span></span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">320</span> Feb <span class="number">6</span> <span class="number">08</span>:<span class="number">23</span> index.html</span><br><span class="line">root@<span class="number">7</span>fbe515cc1ee:/<span class="comment"># cat /usr/share/nginx/html/index.html</span></span><br><span class="line"><!DOCTYPE html></span><br><span class="line"><html></span><br><span class="line"><head></span><br><span class="line"><title>Welcome to nginx!</title></span><br><span class="line"><style></span><br><span class="line"> body {</span><br><span class="line"> width: <span class="number">35</span>em;</span><br><span class="line"> margin: <span class="number">0</span> auto;</span><br><span class="line"> font-family: Tahoma, Verdana, Arial, sans-serif;</span><br><span class="line"> }</span><br><span class="line"></style></span><br><span class="line"></head></span><br><span class="line"><body></span><br><span class="line"></span><br><span class="line"><p id=<span class="string">"p1"</span>>Hello Funs</p></span><br><span class="line"><a href=<span class="string">"https://sxwanggit126.github.io/archives/"</span>>my blog</a></span><br><span class="line"></body></span><br><span class="line"></html></span><br><span class="line">root@<span class="number">7</span>fbe515cc1ee:/<span class="comment">#</span></span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">总结：</span><br><span class="line">docker 目的就是 简化 </span><br><span class="line"></span><br><span class="line">container 容器实例 其实就是一个ps的进程 containerd-shim，那么只能运行一个服务</span><br><span class="line">这句话什么意思呢？</span><br><span class="line"> 就是 加入你要部署一个 伪分布式的 Hadoop</span><br><span class="line"> nn，dn 这两个 都各自是一个进程 </span><br><span class="line"> 也就是说 你用docker 需要 部署两次 一个是 nn 一个是 dn</span><br></pre></td></tr></table></figure></div>
<p><strong>Mysql</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><span class="line">[root@hadoop101 ~]<span class="comment"># docker pull mysql:5.7</span></span><br><span class="line"><span class="number">5.7</span>: Pulling from library/mysql</span><br><span class="line"><span class="number">619014</span>d83c02: Pull complete </span><br><span class="line"><span class="number">9</span>ced578c3a5f: Pull complete </span><br><span class="line"><span class="number">731</span>f6e13d8ea: Pull complete </span><br><span class="line"><span class="number">3</span>c183de42679: Pull complete </span><br><span class="line"><span class="number">6</span>de69b5c2f3c: Pull complete </span><br><span class="line"><span class="number">00</span>f0a4086406: Pull complete </span><br><span class="line"><span class="number">84</span>d93aea836d: Pull complete </span><br><span class="line">e2dd1f3e5a3d: Pull complete </span><br><span class="line"><span class="number">98</span>fb37478ee6: Pull complete </span><br><span class="line"><span class="number">57</span>eb21662ebf: Pull complete </span><br><span class="line">e95057f0a050: Pull complete </span><br><span class="line">Digest: sha256:cf6899e980c38071a78ded028de40e65286bfbbb746b97617ac4c9a84c4e812d</span><br><span class="line">Status: Downloaded newer image <span class="keyword">for</span> mysql:<span class="number">5.7</span></span><br><span class="line">docker.io/library/mysql:<span class="number">5.7</span></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker images</span></span><br><span class="line">REPOSITORY TAG IMAGE ID CREATED SIZE</span><br><span class="line">nginx latest <span class="number">2073</span>e0bcb60e <span class="number">4</span> days ago <span class="number">127</span>MB</span><br><span class="line">mysql <span class="number">5.7</span> c4f186b9e038 <span class="number">4</span> days ago <span class="number">435</span>MB</span><br><span class="line">apache/kudu latest e1c6e5b62086 <span class="number">2</span> months ago <span class="number">476</span>MB</span><br><span class="line">hello-world latest fce289e99eb9 <span class="number">13</span> months ago <span class="number">1.84</span>kB</span><br><span class="line">kunickiaj/kudu latest c30c95f5fe58 <span class="number">17</span> months ago <span class="number">896</span>MB</span><br><span class="line">docker/compose <span class="number">1.8</span>.<span class="number">0</span> <span class="number">89188432</span>ef03 <span class="number">3</span> years ago <span class="number">59.1</span>MB</span><br><span class="line">[root@hadoop101 ~]<span class="comment">#</span></span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200206164751677.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">3</span>.如何去运行 看上面图片</span><br><span class="line"></span><br><span class="line">docker run \</span><br><span class="line">--name doublehappy-mysql1 \</span><br><span class="line">-e MYSQL_ROOT_PASSWORD=<span class="number">123456</span> \</span><br><span class="line">-d mysql:<span class="number">5.7</span></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">-e 指的是 环境变量</span><br><span class="line"> 这个密码怎么传过去的呢？一会解释</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker run \</span></span><br><span class="line">> --name doublehappy-mysql1 \</span><br><span class="line">> -e MYSQL_ROOT_PASSWORD=<span class="number">123456</span> \</span><br><span class="line">> -d mysql:<span class="number">5.7</span></span><br><span class="line">ed9c2cb2591cc8fa85af1c14497bed28c4da9b8c94e94e50e5c5b89ff93154cc</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker ps</span></span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">ed9c2cb2591c mysql:<span class="number">5.7</span> <span class="string">"docker-entrypoint.s…"</span> <span class="number">45</span> seconds ago Up <span class="number">44</span> seconds <span class="number">3306</span>/tcp, <span class="number">33060</span>/tcp doublehappy-mysql1</span><br><span class="line"><span class="number">7</span>fbe515cc1ee nginx:latest <span class="string">"nginx -g 'daemon of…"</span> <span class="number">29</span> minutes ago Up <span class="number">29</span> minutes <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8081</span>-><span class="number">80</span>/tcp doublehappy-nginx2</span><br><span class="line">d4267c461c6c nginx:latest <span class="string">"nginx -g 'daemon of…"</span> About an hour ago Up About an hour <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8080</span>-><span class="number">80</span>/tcp doublehappy-nginx1</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="number">3306</span>/tcp, <span class="number">33060</span>/tcp:</span><br><span class="line"> 表示多个端口 对外提供 映射服务的</span><br><span class="line"></span><br><span class="line">这个没有箭头 说明外面是 访问不了的 </span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker exec -it doublehappy-mysql1 /bin/bash</span></span><br><span class="line">root@ed9c2cb2591c:/<span class="comment"># which ps</span></span><br><span class="line">root@ed9c2cb2591c:/<span class="comment"># </span></span><br><span class="line">root@ed9c2cb2591c:/<span class="comment"># </span></span><br><span class="line">root@ed9c2cb2591c:/<span class="comment"># mysql -uroot -p123456</span></span><br><span class="line">mysql: [Warning] Using a password on the command line interface can be insecure.</span><br><span class="line">Welcome to the MySQL monitor. Commands <span class="keyword">end</span> with ; or \g.</span><br><span class="line">Your MySQL connection id is <span class="number">2</span></span><br><span class="line">Server version: <span class="number">5.7</span>.<span class="number">29</span> MySQL Community Server (GPL)</span><br><span class="line"></span><br><span class="line">Copyright (c) <span class="number">2000</span>, <span class="number">2020</span>, Oracle and/or its affiliates. All rights reserved.</span><br><span class="line"></span><br><span class="line">Oracle is a registered trademark of Oracle Corporation and/or its</span><br><span class="line">affiliates. Other names may be trademarks of their respective</span><br><span class="line">owners.</span><br><span class="line"></span><br><span class="line">Type <span class="string">'help;'</span> or <span class="string">'\h'</span> <span class="keyword">for</span> help. Type <span class="string">'\c'</span> to clear the current input statement.</span><br><span class="line"></span><br><span class="line">mysql></span><br><span class="line"> +--------------------+</span><br><span class="line">| Database |</span><br><span class="line">+--------------------+</span><br><span class="line">| information_schema |</span><br><span class="line">| mysql |</span><br><span class="line">| performance_schema |</span><br><span class="line">| sys |</span><br><span class="line">+--------------------+</span><br><span class="line"><span class="number">4</span> rows <span class="keyword">in</span> set (<span class="number">0.00</span> sec)</span><br><span class="line"></span><br><span class="line">mysql> </span><br><span class="line"></span><br><span class="line"></span><br><span class="line">如何对外访问？？</span><br><span class="line"> 之前的 nginx 是 命令 加一个 -p</span><br><span class="line"></span><br><span class="line">docker run \</span><br><span class="line">--name doublehappy-mysql2 \</span><br><span class="line">-e MYSQL_ROOT_PASSWORD=<span class="number">123456</span> \</span><br><span class="line">-p <span class="number">33060</span>:<span class="number">3306</span> \</span><br><span class="line">-d mysql:<span class="number">5.7</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker run \</span></span><br><span class="line">> --name doublehappy-mysql2 \</span><br><span class="line">> -e MYSQL_ROOT_PASSWORD=<span class="number">123456</span> \</span><br><span class="line">> -p <span class="number">33060</span>:<span class="number">3306</span> \</span><br><span class="line">> -d mysql:<span class="number">5.7</span></span><br><span class="line">b66ce92bfa4f5c53a588a41e015bb7b045eaa602df5cf0d98c40bec7c3bd6571</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># docker ps</span></span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">b66ce92bfa4f mysql:<span class="number">5.7</span> <span class="string">"docker-entrypoint.s…"</span> <span class="number">13</span> seconds ago Up <span class="number">12</span> seconds <span class="number">33060</span>/tcp, <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">33060</span>-><span class="number">3306</span>/tcp doublehappy-mysql2</span><br><span class="line">ed9c2cb2591c mysql:<span class="number">5.7</span> <span class="string">"docker-entrypoint.s…"</span> <span class="number">6</span> minutes ago Up <span class="number">6</span> minutes <span class="number">3306</span>/tcp, <span class="number">33060</span>/tcp doublehappy-mysql1</span><br><span class="line"><span class="number">7</span>fbe515cc1ee nginx:latest <span class="string">"nginx -g 'daemon of…"</span> <span class="number">35</span> minutes ago Up <span class="number">35</span> minutes <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8081</span>-><span class="number">80</span>/tcp doublehappy-nginx2</span><br><span class="line">d4267c461c6c nginx:latest <span class="string">"nginx -g 'daemon of…"</span> About an hour ago Up About an hour <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">8080</span>-><span class="number">80</span>/tcp doublehappy-nginx1</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line">说明：</span><br><span class="line"><span class="number">33060</span>/tcp, <span class="number">0.0</span>.<span class="number">0.0</span>:<span class="number">33060</span>-><span class="number">3306</span>/tcp 映射成功 </span><br><span class="line"></span><br><span class="line">ok 我们 用 海迪链接一下</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200206170328683.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">[root@hadoop101 ~]<span class="comment"># ps -ef | grep docker</span></span><br><span class="line">root <span class="number">6599</span> <span class="number">1</span> <span class="number">0</span> <span class="number">14</span>:<span class="number">13</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">19</span> /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock</span><br><span class="line">root <span class="number">11669</span> <span class="number">6599</span> <span class="number">0</span> <span class="number">15</span>:<span class="number">49</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> /usr/bin/docker-proxy -proto tcp -host-ip <span class="number">0.0</span>.<span class="number">0.0</span> -host-port <span class="number">8080</span> -container-ip <span class="number">172.17</span>.<span class="number">0.2</span> -container-port <span class="number">80</span></span><br><span class="line">root <span class="number">11701</span> <span class="number">6598</span> <span class="number">0</span> <span class="number">15</span>:<span class="number">49</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> containerd-shim -namespace moby -workdir /var/lib/containerd/io.containerd.runtime.v1.linux/moby/d4267c461c6cb0cfaeac3a2e4037f6ad1fbb57bfe31264888ff1e4ca220e5f8b -address /run/containerd/containerd.sock -containerd-binary /usr/bin/containerd -runtime-root /var/run/docker/runtime-runc</span><br><span class="line">root <span class="number">13582</span> <span class="number">3143</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">01</span> pts/<span class="number">1</span> <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> docker exec -it d4267c461c6c /bin/bash</span><br><span class="line">root <span class="number">13909</span> <span class="number">6599</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">20</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> /usr/bin/docker-proxy -proto tcp -host-ip <span class="number">0.0</span>.<span class="number">0.0</span> -host-port <span class="number">8081</span> -container-ip <span class="number">172.17</span>.<span class="number">0.3</span> -container-port <span class="number">80</span></span><br><span class="line">root <span class="number">13925</span> <span class="number">6598</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">20</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> containerd-shim -namespace moby -workdir /var/lib/containerd/io.containerd.runtime.v1.linux/moby/<span class="number">7</span>fbe515cc1ee97390b66f686f0119232c9f329898b9f57bebabdfd89f23fd641 -address /run/containerd/containerd.sock -containerd-binary /usr/bin/containerd -runtime-root /var/run/docker/runtime-runc</span><br><span class="line">root <span class="number">14128</span> <span class="number">13798</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">25</span> pts/<span class="number">0</span> <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> docker exec -it doublehappy-nginx2 /bin/bash</span><br><span class="line">root <span class="number">16274</span> <span class="number">6598</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">49</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> containerd-shim -namespace moby -workdir /var/lib/containerd/io.containerd.runtime.v1.linux/moby/ed9c2cb2591cc8fa85af1c14497bed28c4da9b8c94e94e50e5c5b89ff93154cc -address /run/containerd/containerd.sock -containerd-binary /usr/bin/containerd -runtime-root /var/run/docker/runtime-runc</span><br><span class="line">root <span class="number">16786</span> <span class="number">6599</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">55</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> /usr/bin/docker-proxy -proto tcp -host-ip <span class="number">0.0</span>.<span class="number">0.0</span> -host-port <span class="number">33060</span> -container-ip <span class="number">172.17</span>.<span class="number">0.5</span> -container-port <span class="number">3306</span></span><br><span class="line">root <span class="number">16810</span> <span class="number">6598</span> <span class="number">0</span> <span class="number">16</span>:<span class="number">55</span> ? <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> containerd-shim -namespace moby -workdir /var/lib/containerd/io.containerd.runtime.v1.linux/moby/b66ce92bfa4f5c53a588a41e015bb7b045eaa602df5cf0d98c40bec7c3bd6571 -address /run/containerd/containerd.sock -containerd-binary /usr/bin/containerd -runtime-root /var/run/docker/runtime-runc</span><br><span class="line">root <span class="number">17188</span> <span class="number">14323</span> <span class="number">0</span> <span class="number">17</span>:<span class="number">04</span> pts/<span class="number">2</span> <span class="number">00</span>:<span class="number">00</span>:<span class="number">00</span> grep --color=auto docker</span><br><span class="line">[root@hadoop101 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">runtime-runc 有四个 </span><br><span class="line"></span><br><span class="line"></span><br><span class="line">我们进程 运行了 四个 正确</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> 是个进程 </span><br><span class="line"> 那么 mysql 的数据存放在哪里呢？</span><br><span class="line"> 会不会随着进程的kill 而数据丢失呢？</span><br></pre></td></tr></table></figure></div>
<h2 id="docker-架构"><a href="#docker-架构" class="headerlink" title="docker 架构"></a>docker 架构</h2><p><img src="https://img-blog.csdnimg.cn/20200206170932854.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><a href="https://docs.docker.com/engine/docker-overview/" target="_blank" rel="noopener">架构</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">按照图片 线 走</span><br></pre></td></tr></table></figure></div>
<h2 id="docker-如何构建自己的镜像"><a href="#docker-如何构建自己的镜像" class="headerlink" title="docker 如何构建自己的镜像"></a>docker 如何构建自己的镜像</h2><p><a href="https://github.com/docker-library/mysql/blob/6952c5d5a9889311157362c528d65dc2e37ff660/5.7/Dockerfile" target="_blank" rel="noopener">以官方 mysql为例</a><br><a href="https://github.com/docker-library/mysql/blob/6952c5d5a9889311157362c528d65dc2e37ff660/5.7/docker-entrypoint.sh" target="_blank" rel="noopener">mysql Dockerfile 脚本 ： 相当于入口</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">1</span>.找</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 要构建我们的 Dockerfile 就是把官网的 拿来</span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># mkdir mysql5.7</span></span><br><span class="line">[root@hadoop101 ~]<span class="comment"># cd mysql5.7/</span></span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># vi Dockerfile</span></span><br><span class="line">FROM debian:stretch-slim</span><br><span class="line"></span><br><span class="line">RUN groupadd -r mysql && useradd -r -g mysql mysql</span><br><span class="line"></span><br><span class="line">RUN apt-get update && apt-get install -y --no-install-recommends gnupg dirmngr && rm -rf /var/lib/apt/lists/*</span><br><span class="line"></span><br><span class="line"><span class="comment"># add gosu for easy step-down from root</span></span><br><span class="line">ENV GOSU_VERSION <span class="number">1.7</span></span><br><span class="line">RUN set -x \</span><br><span class="line"> && export GNUPGHOME=<span class="string">"$(mktemp -d)"</span> \</span><br><span class="line"> && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \</span><br><span class="line"> && gpgconf --kill all \</span><br><span class="line"> && rm -rf <span class="string">"<span class="variable">$GNUPGHOME</span>"</span> /usr/local/bin/gosu.asc \</span><br><span class="line"> && chmod +x /usr/local/bin/gosu \</span><br><span class="line"> && gosu nobody true \</span><br><span class="line"> && apt-get purge -y --auto-remove ca-certificates wget</span><br><span class="line"></span><br><span class="line">RUN mkdir /docker-entrypoint-initdb.d</span><br><span class="line"></span><br><span class="line">RUN apt-get update && apt-get install -y --no-install-recommends \</span><br><span class="line"><span class="comment"># for MYSQL_RANDOM_ROOT_PASSWORD</span></span><br><span class="line"> pwgen \</span><br><span class="line"><span class="comment"># for mysql_ssl_rsa_setup</span></span><br><span class="line"> openssl \</span><br><span class="line"><span class="comment"># File::Basename</span></span><br><span class="line"><span class="comment"># File::Copy</span></span><br><span class="line"><span class="comment"># Sys::Hostname</span></span><br><span class="line"> && rm -rf /var/lib/apt/lists/*</span><br><span class="line"></span><br><span class="line">RUN set -ex; \</span><br><span class="line"><span class="comment"># gpg: key 5072E1F5: public key "MySQL Release Engineering <[email protected]>" imported</span></span><br><span class="line"> key=<span class="string">'A4A9406876FCBD3C456770C88C718D3B5072E1F5'</span>; \</span><br><span class="line"> export GNUPGHOME=<span class="string">"$(mktemp -d)"</span>; \</span><br><span class="line"> gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys <span class="string">"<span class="variable">$key</span>"</span>; \</span><br><span class="line"> gpg --batch --export <span class="string">"<span class="variable">$key</span>"</span> > /etc/apt/trusted.gpg.d/mysql.gpg; \</span><br><span class="line"> gpgconf --kill all; \</span><br><span class="line"> rm -rf <span class="string">"<span class="variable">$GNUPGHOME</span>"</span>; \</span><br><span class="line">ENV MYSQL_MAJOR <span class="number">5.7</span></span><br><span class="line">ENV MYSQL_VERSION <span class="number">5.7</span>.<span class="number">29</span>-<span class="number">1</span>debian9</span><br><span class="line"></span><br><span class="line">RUN echo <span class="string">"deb http://repo.mysql.com/apt/debian/ stretch mysql-${MYSQL_MAJOR}"</span> > /etc/apt/sources.list.d/mysql.list</span><br><span class="line"></span><br><span class="line"><span class="comment"># also, we set debconf keys to make APT a little quieter</span></span><br><span class="line">RUN { \</span><br><span class="line"> echo mysql-community-server mysql-community-server/data-dir select <span class="string">''</span>; \</span><br><span class="line"> echo mysql-community-server mysql-community-server/root-pass password <span class="string">''</span>; \</span><br><span class="line"> echo mysql-community-server mysql-community-server/re-root-pass password <span class="string">''</span>; \</span><br><span class="line"> echo mysql-community-server mysql-community-server/remove-test-db select false; \</span><br><span class="line"> } | debconf-set-selections \</span><br><span class="line"> && apt-get update && apt-get install -y mysql-server=<span class="string">"${MYSQL_VERSION}"</span> && rm -rf /var/lib/apt/lists/* \</span><br><span class="line"> && rm -rf /var/lib/mysql && mkdir -p /var/lib/mysql /var/run/mysqld \</span><br><span class="line"> && chown -R mysql:mysql /var/lib/mysql /var/run/mysqld \</span><br><span class="line"><span class="comment"># ensure that /var/run/mysqld (used for socket and lock files) is writable regardless of the UID our mysqld instan</span></span><br><span class="line">ce ends up having at runtime</span><br><span class="line"> && chmod <span class="number">777</span> /var/run/mysqld \</span><br><span class="line"><span class="comment"># comment out a few problematic configuration values</span></span><br><span class="line"> && find /etc/mysql/ -name <span class="string">'*.cnf'</span> -print0 \</span><br><span class="line"> | xargs -<span class="number">0</span> grep -lZE <span class="string">'^(bind-address|log)'</span> \</span><br><span class="line"> | xargs -rt -<span class="number">0</span> sed -Ei <span class="string">'s/^(bind-address|log)/#&/'</span> \</span><br><span class="line"><span class="comment"># don't reverse lookup hostnames, they are usually another container</span></span><br><span class="line"> && echo <span class="string">'[mysqld]\nskip-host-cache\nskip-name-resolve'</span> > /etc/mysql/conf.d/docker.cnf</span><br><span class="line"></span><br><span class="line">VOLUME /var/lib/mysql</span><br><span class="line"></span><br><span class="line">COPY docker-entrypoint.sh /usr/local/bin/</span><br><span class="line">RUN ln -s usr/local/bin/docker-entrypoint.sh /entrypoint.sh <span class="comment"># backwards compat</span></span><br><span class="line">ENTRYPOINT [<span class="string">"docker-entrypoint.sh"</span>]</span><br><span class="line"></span><br><span class="line">EXPOSE <span class="number">3306</span> <span class="number">33060</span></span><br><span class="line">CMD [<span class="string">"mysqld"</span>]</span><br><span class="line"><span class="string">"Dockerfile"</span> [New] <span class="number">80</span>L, <span class="number">3803</span>C written</span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 一般Dockerfile 都有 shell脚本 看看 Dockerfile docker-entrypoint.sh</span><br><span class="line"> 这个脚本就是入口</span><br><span class="line"></span><br><span class="line">把这个脚本也做好 </span><br><span class="line"></span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># ll</span></span><br><span class="line">total <span class="number">20</span></span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">12713</span> Feb <span class="number">6</span> <span class="number">17</span>:<span class="number">23</span> docker-entrypoint.sh</span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">3803</span> Feb <span class="number">6</span> <span class="number">17</span>:<span class="number">18</span> Dockerfile</span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"><span class="number">2</span>.命令构建</span><br><span class="line"></span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># ll</span></span><br><span class="line">total <span class="number">20</span></span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">12713</span> Feb <span class="number">6</span> <span class="number">17</span>:<span class="number">23</span> docker-entrypoint.sh</span><br><span class="line">-rw-r--r-- <span class="number">1</span> root root <span class="number">3803</span> Feb <span class="number">6</span> <span class="number">17</span>:<span class="number">18</span> Dockerfile</span><br><span class="line"></span><br><span class="line">[root@hadoop101 mysql5.<span class="number">7</span>]<span class="comment"># docker build ./ -t mysql:doublehappy5.7</span></span><br><span class="line">Sending build context to Docker daemon <span class="number">18.94</span>kB</span><br><span class="line">Step <span class="number">1</span>/<span class="number">18</span> : FROM debian:stretch-slim</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 建议 用国外的 服务器 可以翻墙的 要不然真的很慢</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line"><span class="number">3</span>.解读Dockerfile：</span><br><span class="line">FROM 基础镜像</span><br><span class="line"> eg:</span><br><span class="line"> FROM debian:stretch-slim</span><br><span class="line"> 系统名字: 镜像的名字</span><br><span class="line"> 注意：debian 这个系统比较轻量级 centos 比较重 所以 debian 最终构建的 镜像比较小</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">RUN 在基础镜像的Linux操作命令</span><br><span class="line"> eg：RUN groupadd -r mysql && useradd -r -g mysql mysql</span><br><span class="line"></span><br><span class="line">apt-get ==》yum </span><br><span class="line"></span><br><span class="line">ENV 环境变量</span><br><span class="line"> k=v <span class="variable">$k</span></span><br><span class="line"></span><br><span class="line">VOLUME 定义匿名卷 !!!! 就是 文件的目录 这个很重要</span><br><span class="line"></span><br><span class="line">COPY 复制文本</span><br><span class="line"> 源路径 目标路径</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">ENTRYPOINT [<span class="string">"docker-entrypoint.sh"</span>]</span><br><span class="line"> 设定容器启动时第一个运行的命令及参数</span><br><span class="line"> 也就是说 这个shell 脚本里面 可以接收外部的 参数 </span><br><span class="line"></span><br><span class="line">EXPOSE <span class="number">3306</span> <span class="number">33060</span> </span><br><span class="line"> 仅作为容器的服务使用什么端口号，且并不会主动在宿主Linux上自动映射端口</span><br><span class="line"></span><br><span class="line">CMD [<span class="string">"mysqld"</span>]</span><br><span class="line"> 运行服务</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">总结：</span><br><span class="line"> 毕竟 这是官网的</span><br><span class="line"> 你生产上 你要根据你自己的需求 来构建镜像 </span><br><span class="line"> Dockfile 如何编写 和 shell 脚本 都是要你自己来写哒</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">shell 脚本：</span><br><span class="line"></span><br><span class="line">/bin/bash</span><br><span class="line"></span><br><span class="line">f2() {</span><br><span class="line"> k2=v2</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">f1() {</span><br><span class="line"> echo $k2</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">_main() {</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> shell脚本是 从上往下加载的</span><br></pre></td></tr></table></figure></div>
<h2 id="停止"><a href="#停止" class="headerlink" title="停止"></a>停止</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">[root@hadoop101 ~]# docker ps</span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">3b4168987e1b 657af8c03255 "/bin/sh -c 'set -x …" 5 minutes ago Up 5 minutes agitated_sammet</span><br><span class="line">b66ce92bfa4f mysql:5.7 "docker-entrypoint.s…" 2 hours ago Up 2 hours 33060/tcp, 0.0.0.0:33060->3306/tcp doublehappy-mysql2</span><br><span class="line">ed9c2cb2591c mysql:5.7 "docker-entrypoint.s…" 2 hours ago Up 2 hours 3306/tcp, 33060/tcp doublehappy-mysql1</span><br><span class="line">7fbe515cc1ee nginx:latest "nginx -g 'daemon of…" 3 hours ago Up 3 hours 0.0.0.0:8081->80/tcp doublehappy-nginx2</span><br><span class="line">d4267c461c6c nginx:latest "nginx -g 'daemon of…" 3 hours ago Up 3 hours 0.0.0.0:8080->80/tcp doublehappy-nginx1</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[root@hadoop101 ~]# docker stop doublehappy-mysql1</span><br><span class="line">doublehappy-mysql1</span><br><span class="line">[root@hadoop101 ~]# docker ps</span><br><span class="line">CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES</span><br><span class="line">3b4168987e1b 657af8c03255 "/bin/sh -c 'set -x …" 6 minutes ago Up 6 minutes agitated_sammet</span><br><span class="line">b66ce92bfa4f mysql:5.7 "docker-entrypoint.s…" 2 hours ago Up 2 hours 33060/tcp, 0.0.0.0:33060->3306/tcp doublehappy-mysql2</span><br><span class="line">7fbe515cc1ee nginx:latest "nginx -g 'daemon of…" 3 hours ago Up 3 hours 0.0.0.0:8081->80/tcp doublehappy-nginx2</span><br><span class="line">d4267c461c6c nginx:latest "nginx -g 'daemon of…" 3 hours ago Up 3 hours 0.0.0.0:8080->80/tcp doublehappy-nginx1</span><br><span class="line">[root@hadoop101 ~]#</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="POWERSHELL"><figure class="iseeu highlight /powershell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br></pre></td><td class="code"><pre><span class="line">[root@hadoop101 ~]<span class="comment"># docker --help</span></span><br><span class="line"></span><br><span class="line">Usage: docker [OPTIONS] COMMAND</span><br><span class="line"></span><br><span class="line">A self-sufficient runtime <span class="keyword">for</span> containers</span><br><span class="line"></span><br><span class="line">Options:</span><br><span class="line"> --config string Location of client config files (default <span class="string">"/root/.docker"</span>)</span><br><span class="line"> -c, --context string Name of the context to use to connect to the daemon (overrides DOCKER_HOST env</span><br><span class="line"> var and default context set with <span class="string">"docker context use"</span>)</span><br><span class="line"> -D, --debug Enable debug mode</span><br><span class="line"> -H, --host list Daemon socket(s) to connect to</span><br><span class="line"> -l, --log-level string Set the logging level (<span class="string">"debug"</span>|<span class="string">"info"</span>|<span class="string">"warn"</span>|<span class="string">"error"</span>|<span class="string">"fatal"</span>) (default <span class="string">"info"</span>)</span><br><span class="line"> --tls Use TLS; implied by --tlsverify</span><br><span class="line"> --tlscacert string Trust certs signed only by this CA (default <span class="string">"/root/.docker/ca.pem"</span>)</span><br><span class="line"> --tlscert string Path to TLS certificate file (default <span class="string">"/root/.docker/cert.pem"</span>)</span><br><span class="line"> --tlskey string Path to TLS key file (default <span class="string">"/root/.docker/key.pem"</span>)</span><br><span class="line"> --tlsverify Use TLS and verify the remote</span><br><span class="line"> -v, --version Print version information and quit</span><br><span class="line"></span><br><span class="line">Management Commands:</span><br><span class="line"> builder Manage builds</span><br><span class="line"> config Manage Docker configs</span><br><span class="line"> container Manage containers</span><br><span class="line"> context Manage contexts</span><br><span class="line"> engine Manage the docker engine</span><br><span class="line"> image Manage images</span><br><span class="line"> network Manage networks</span><br><span class="line"> node Manage Swarm nodes</span><br><span class="line"> plugin Manage plugins</span><br><span class="line"> secret Manage Docker secrets</span><br><span class="line"> service Manage services</span><br><span class="line"> stack Manage Docker stacks</span><br><span class="line"> swarm Manage Swarm</span><br><span class="line"> system Manage Docker</span><br><span class="line"> trust Manage trust on Docker images</span><br><span class="line"> volume Manage volumes</span><br><span class="line"></span><br><span class="line">Commands:</span><br><span class="line"> attach Attach local standard input, output, and error streams to a running container</span><br><span class="line"> build Build an image from a Dockerfile</span><br><span class="line"> commit Create a new image from a container<span class="string">'s changes</span></span><br><span class="line"><span class="string"> cp Copy files/folders between a container and the local filesystem</span></span><br><span class="line"><span class="string"> create Create a new container</span></span><br><span class="line"><span class="string"> diff Inspect changes to files or directories on a container'</span>s filesystem</span><br><span class="line"> events Get real time events from the server</span><br><span class="line"> exec Run a command <span class="keyword">in</span> a running container</span><br><span class="line"> export Export a container<span class="string">'s filesystem as a tar archive</span></span><br><span class="line"><span class="string"> history Show the history of an image</span></span><br><span class="line"><span class="string"> images List images</span></span><br><span class="line"><span class="string"> import Import the contents from a tarball to create a filesystem image</span></span><br><span class="line"><span class="string"> info Display system-wide information</span></span><br><span class="line"><span class="string"> inspect Return low-level information on Docker objects</span></span><br><span class="line"><span class="string"> kill Kill one or more running containers</span></span><br><span class="line"><span class="string"> load Load an image from a tar archive or STDIN</span></span><br><span class="line"><span class="string"> login Log in to a Docker registry</span></span><br><span class="line"><span class="string"> logout Log out from a Docker registry</span></span><br><span class="line"><span class="string"> logs Fetch the logs of a container</span></span><br><span class="line"><span class="string"> pause Pause all processes within one or more containers</span></span><br><span class="line"><span class="string"> port List port mappings or a specific mapping for the container</span></span><br><span class="line"><span class="string"> ps List containers</span></span><br><span class="line"><span class="string"> pull Pull an image or a repository from a registry</span></span><br><span class="line"><span class="string"> push Push an image or a repository to a registry</span></span><br><span class="line"><span class="string"> rename Rename a container</span></span><br><span class="line"><span class="string"> restart Restart one or more containers</span></span><br><span class="line"><span class="string"> rm Remove one or more containers</span></span><br><span class="line"><span class="string"> rmi Remove one or more images</span></span><br><span class="line"><span class="string"> run Run a command in a new container</span></span><br><span class="line"><span class="string"> save Save one or more images to a tar archive (streamed to STDOUT by default)</span></span><br><span class="line"><span class="string"> search Search the Docker Hub for images</span></span><br><span class="line"><span class="string"> start Start one or more stopped containers</span></span><br><span class="line"><span class="string"> stats Display a live stream of container(s) resource usage statistics</span></span><br><span class="line"><span class="string"> stop Stop one or more running containers</span></span><br><span class="line"><span class="string"> tag Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE</span></span><br><span class="line"><span class="string"> top Display the running processes of a container</span></span><br><span class="line"><span class="string"> unpause Unpause all processes within one or more containers</span></span><br><span class="line"><span class="string"> update Update configuration of one or more containers</span></span><br><span class="line"><span class="string"> version Show the Docker version information</span></span><br><span class="line"><span class="string"> wait Block until one or more containers stop, then print their exit codes</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">Run '</span>docker COMMAND --help<span class="string">' for more information on a command.</span></span><br><span class="line"><span class="string">[root@hadoop101 ~]#</span></span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-k8s-Spark-doublehappy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/04/27/k8s-Spark-doublehappy/">k8s-Spark-doublehappy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/04/27/k8s-Spark-doublehappy/" class="article-date">
<time datetime="2020-04-27T09:07:46.187Z" itemprop="datePublished">2020-04-27</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="Spark-ON-K8S"><a href="#Spark-ON-K8S" class="headerlink" title="Spark ON K8S"></a>Spark ON K8S</h2><p><a href="http://spark.apache.org/docs/latest/running-on-kubernetes.html" target="_blank" rel="noopener">Spark on K8s</a><br><img src="https://img-blog.csdnimg.cn/20200211152754486.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTU1NzUxMg==,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="BASH"><figure class="iseeu highlight /bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br><span class="line">245</span><br><span class="line">246</span><br><span class="line">247</span><br><span class="line">248</span><br><span class="line">249</span><br><span class="line">250</span><br><span class="line">251</span><br><span class="line">252</span><br><span class="line">253</span><br><span class="line">254</span><br><span class="line">255</span><br><span class="line">256</span><br><span class="line">257</span><br><span class="line">258</span><br><span class="line">259</span><br><span class="line">260</span><br><span class="line">261</span><br><span class="line">262</span><br><span class="line">263</span><br><span class="line">264</span><br><span class="line">265</span><br><span class="line">266</span><br><span class="line">267</span><br><span class="line">268</span><br><span class="line">269</span><br><span class="line">270</span><br><span class="line">271</span><br><span class="line">272</span><br><span class="line">273</span><br><span class="line">274</span><br><span class="line">275</span><br><span class="line">276</span><br><span class="line">277</span><br><span class="line">278</span><br><span class="line">279</span><br><span class="line">280</span><br><span class="line">281</span><br><span class="line">282</span><br><span class="line">283</span><br><span class="line">284</span><br><span class="line">285</span><br><span class="line">286</span><br><span class="line">287</span><br><span class="line">288</span><br><span class="line">289</span><br><span class="line">290</span><br><span class="line">291</span><br><span class="line">292</span><br><span class="line">293</span><br><span class="line">294</span><br><span class="line">295</span><br><span class="line">296</span><br><span class="line">297</span><br><span class="line">298</span><br><span class="line">299</span><br><span class="line">300</span><br><span class="line">301</span><br><span class="line">302</span><br><span class="line">303</span><br><span class="line">304</span><br><span class="line">305</span><br><span class="line">306</span><br><span class="line">307</span><br><span class="line">308</span><br><span class="line">309</span><br><span class="line">310</span><br><span class="line">311</span><br><span class="line">312</span><br><span class="line">313</span><br><span class="line">314</span><br><span class="line">315</span><br><span class="line">316</span><br><span class="line">317</span><br><span class="line">318</span><br><span class="line">319</span><br><span class="line">320</span><br><span class="line">321</span><br><span class="line">322</span><br><span class="line">323</span><br><span class="line">324</span><br><span class="line">325</span><br><span class="line">326</span><br><span class="line">327</span><br><span class="line">328</span><br><span class="line">329</span><br><span class="line">330</span><br><span class="line">331</span><br><span class="line">332</span><br><span class="line">333</span><br><span class="line">334</span><br><span class="line">335</span><br><span class="line">336</span><br><span class="line">337</span><br><span class="line">338</span><br><span class="line">339</span><br><span class="line">340</span><br><span class="line">341</span><br><span class="line">342</span><br><span class="line">343</span><br><span class="line">344</span><br><span class="line">345</span><br><span class="line">346</span><br><span class="line">347</span><br><span class="line">348</span><br><span class="line">349</span><br><span class="line">350</span><br><span class="line">351</span><br><span class="line">352</span><br><span class="line">353</span><br><span class="line">354</span><br><span class="line">355</span><br><span class="line">356</span><br><span class="line">357</span><br><span class="line">358</span><br><span class="line">359</span><br><span class="line">360</span><br><span class="line">361</span><br><span class="line">362</span><br><span class="line">363</span><br><span class="line">364</span><br><span class="line">365</span><br><span class="line">366</span><br><span class="line">367</span><br><span class="line">368</span><br><span class="line">369</span><br><span class="line">370</span><br><span class="line">371</span><br><span class="line">372</span><br><span class="line">373</span><br><span class="line">374</span><br><span class="line">375</span><br><span class="line">376</span><br><span class="line">377</span><br><span class="line">378</span><br><span class="line">379</span><br><span class="line">380</span><br><span class="line">381</span><br><span class="line">382</span><br><span class="line">383</span><br><span class="line">384</span><br><span class="line">385</span><br><span class="line">386</span><br><span class="line">387</span><br><span class="line">388</span><br><span class="line">389</span><br><span class="line">390</span><br><span class="line">391</span><br><span class="line">392</span><br><span class="line">393</span><br><span class="line">394</span><br><span class="line">395</span><br><span class="line">396</span><br><span class="line">397</span><br><span class="line">398</span><br><span class="line">399</span><br><span class="line">400</span><br><span class="line">401</span><br><span class="line">402</span><br><span class="line">403</span><br><span class="line">404</span><br><span class="line">405</span><br><span class="line">406</span><br><span class="line">407</span><br><span class="line">408</span><br><span class="line">409</span><br><span class="line">410</span><br><span class="line">411</span><br><span class="line">412</span><br><span class="line">413</span><br><span class="line">414</span><br><span class="line">415</span><br><span class="line">416</span><br><span class="line">417</span><br><span class="line">418</span><br><span class="line">419</span><br><span class="line">420</span><br><span class="line">421</span><br><span class="line">422</span><br><span class="line">423</span><br><span class="line">424</span><br><span class="line">425</span><br><span class="line">426</span><br><span class="line">427</span><br><span class="line">428</span><br><span class="line">429</span><br><span class="line">430</span><br><span class="line">431</span><br><span class="line">432</span><br><span class="line">433</span><br><span class="line">434</span><br><span class="line">435</span><br><span class="line">436</span><br><span class="line">437</span><br><span class="line">438</span><br><span class="line">439</span><br><span class="line">440</span><br><span class="line">441</span><br><span class="line">442</span><br><span class="line">443</span><br><span class="line">444</span><br><span class="line">445</span><br><span class="line">446</span><br><span class="line">447</span><br><span class="line">448</span><br><span class="line">449</span><br></pre></td><td class="code"><pre><span class="line">k8s 数据存储是 etcd（生产上 这个 etcd 要做集群的）</span><br><span class="line"></span><br><span class="line">The Kubernetes scheduler is currently experimental. </span><br><span class="line">spark on k8s 是实验阶段 所以生产上 没有使用 只是测试 </span><br><span class="line"></span><br><span class="line">环境准备：</span><br><span class="line">1.下载Spark</span><br><span class="line">wget https://archive.apache.org/dist/spark/spark-2.4.3/spark-2.4.3-bin-hadoop2.6.tgz</span><br><span class="line">tar -xzvf spark-2.4.3-bin-hadoop2.6.tgz -C /opt/</span><br><span class="line">mv /opt/spark-2.4.3-bin-hadoop2.6 /opt/spark </span><br><span class="line"><span class="built_in">cd</span> /opt/spark</span><br><span class="line"></span><br><span class="line">注意：一定是 /opt/spark 目录下 ！！！</span><br><span class="line">[root@container01 spark]<span class="comment"># pwd</span></span><br><span class="line">/opt/spark</span><br><span class="line">[root@container01 spark]<span class="comment"># ll</span></span><br><span class="line">total 128</span><br><span class="line">drwxr-xr-x 2 1000 1000 4096 May 1 2019 bin</span><br><span class="line">drwxr-xr-x 2 1000 1000 4096 May 1 2019 conf</span><br><span class="line">drwxr-xr-x 5 1000 1000 4096 May 1 2019 data</span><br><span class="line">drwxr-xr-x 4 1000 1000 4096 May 1 2019 examples</span><br><span class="line">drwxr-xr-x 2 1000 1000 12288 May 1 2019 jars</span><br><span class="line">drwxr-xr-x 4 1000 1000 4096 May 1 2019 kubernetes</span><br><span class="line">-rw-r--r-- 1 1000 1000 21316 May 1 2019 LICENSE</span><br><span class="line">drwxr-xr-x 2 1000 1000 4096 May 1 2019 licenses</span><br><span class="line">-rw-r--r-- 1 1000 1000 42919 May 1 2019 NOTICE</span><br><span class="line">drwxr-xr-x 7 1000 1000 4096 May 1 2019 python</span><br><span class="line">drwxr-xr-x 3 1000 1000 4096 May 1 2019 R</span><br><span class="line">-rw-r--r-- 1 1000 1000 3952 May 1 2019 README.md</span><br><span class="line">-rw-r--r-- 1 1000 1000 164 May 1 2019 RELEASE</span><br><span class="line">drwxr-xr-x 2 1000 1000 4096 May 1 2019 sbin</span><br><span class="line">drwxr-xr-x 2 1000 1000 4096 May 1 2019 yarn</span><br><span class="line">[root@container01 spark]<span class="comment">#</span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">2.构建spark镜像</span><br><span class="line">./bin/docker-image-tool.sh -r 172.21.230.89/doublehappy -t v2.4.3 build</span><br><span class="line">./bin/docker-image-tool.sh -r 172.21.230.89/doublehappy -t v2.4.3 push</span><br><span class="line"></span><br><span class="line">[root@container01 spark]<span class="comment"># ./bin/docker-image-tool.sh -r 172.21.230.89/doublehappy -t v2.4.3 build</span></span><br><span class="line">Sending build context to Docker daemon 259.1MB</span><br><span class="line">Step 1/15 : FROM openjdk:8-alpine</span><br><span class="line">8-alpine: Pulling from library/openjdk</span><br><span class="line">e7c96db7181b: Pull complete </span><br><span class="line">f910a506b6cb: Pull complete </span><br><span class="line">c2274a1a0e27: Pull complete </span><br><span class="line">Digest: sha256:94792824df2df33402f201713f932b58cb9de94a0cd524164a0f2283343547b3</span><br><span class="line">Status: Downloaded newer image <span class="keyword">for</span> openjdk:8-alpine</span><br><span class="line"> ---> a3562aa0b991</span><br><span class="line">Step 2/15 : ARG spark_jars=jars</span><br><span class="line"> ---> Running <span class="keyword">in</span> 1ed1dcbc748b</span><br><span class="line">Removing intermediate container 1ed1dcbc748b</span><br><span class="line"> ---> 4bf840b35abc</span><br><span class="line">Step 3/15 : ARG img_path=kubernetes/dockerfiles</span><br><span class="line"> ---> Running <span class="keyword">in</span> 2e0a10a5eeeb</span><br><span class="line">Removing intermediate container 2e0a10a5eeeb</span><br><span class="line"> ---> e7b176887b3d</span><br><span class="line">Step 4/15 : ARG k8s_tests=kubernetes/tests</span><br><span class="line"> ---> Running <span class="keyword">in</span> 5d78a64b22a9</span><br><span class="line">Removing intermediate container 5d78a64b22a9</span><br><span class="line"> ---> 7dda4f6f30d8</span><br><span class="line">Step 5/15 : RUN <span class="built_in">set</span> -ex && apk upgrade --no-cache && apk add --no-cache bash tini libc6-compat linux-pam nss && mkdir -p /opt/spark && mkdir -p /opt/spark/work-dir && touch /opt/spark/RELEASE && rm /bin/sh && ln -sv /bin/bash /bin/sh && <span class="built_in">echo</span> <span class="string">"auth required pam_wheel.so use_uid"</span> >> /etc/pam.d/su && chgrp root /etc/passwd && chmod ug+rw /etc/passwd</span><br><span class="line"> ---> Running <span class="keyword">in</span> 52cff938c375</span><br><span class="line">+ apk upgrade --no-cache</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz</span><br><span class="line">(1/11) Upgrading musl (1.1.20-r4 -> 1.1.20-r5)</span><br><span class="line">(2/11) Upgrading libcrypto1.1 (1.1.1b-r1 -> 1.1.1d-r2)</span><br><span class="line">(3/11) Upgrading libssl1.1 (1.1.1b-r1 -> 1.1.1d-r2)</span><br><span class="line">(4/11) Upgrading musl-utils (1.1.20-r4 -> 1.1.20-r5)</span><br><span class="line">(5/11) Upgrading libtasn1 (4.13-r0 -> 4.14-r0)</span><br><span class="line">(6/11) Upgrading sqlite-libs (3.26.0-r3 -> 3.28.0-r2)</span><br><span class="line">(7/11) Upgrading libbz2 (1.0.6-r6 -> 1.0.6-r7)</span><br><span class="line">(8/11) Upgrading libjpeg-turbo (1.5.3-r4 -> 1.5.3-r6)</span><br><span class="line">(9/11) Upgrading libcom_err (1.44.5-r0 -> 1.44.5-r2)</span><br><span class="line">(10/11) Upgrading openjdk8-jre-base (8.212.04-r0 -> 8.242.08-r0)</span><br><span class="line">(11/11) Upgrading openjdk8-jre (8.212.04-r0 -> 8.242.08-r0)</span><br><span class="line">Executing busybox-1.29.3-r10.trigger</span><br><span class="line">Executing ca-certificates-20190108-r0.trigger</span><br><span class="line">Executing java-common-0.1-r0.trigger</span><br><span class="line">OK: 103 MiB <span class="keyword">in</span> 54 packages</span><br><span class="line">+ apk add --no-cache bash tini libc6-compat linux-pam nss</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz</span><br><span class="line">(1/8) Installing ncurses-terminfo-base (6.1_p20190105-r0)</span><br><span class="line">(2/8) Installing ncurses-terminfo (6.1_p20190105-r0)</span><br><span class="line">(3/8) Installing ncurses-libs (6.1_p20190105-r0)</span><br><span class="line">(4/8) Installing readline (7.0.003-r1)</span><br><span class="line">(5/8) Installing bash (4.4.19-r1)</span><br><span class="line">Executing bash-4.4.19-r1.post-install</span><br><span class="line">(6/8) Installing libc6-compat (1.1.20-r5)</span><br><span class="line">(7/8) Installing linux-pam (1.3.0-r0)</span><br><span class="line">(8/8) Installing tini (0.18.0-r0)</span><br><span class="line">Executing busybox-1.29.3-r10.trigger</span><br><span class="line">OK: 113 MiB <span class="keyword">in</span> 62 packages</span><br><span class="line">+ mkdir -p /opt/spark</span><br><span class="line">+ mkdir -p /opt/spark/work-dir</span><br><span class="line">+ touch /opt/spark/RELEASE</span><br><span class="line">+ rm /bin/sh</span><br><span class="line">+ ln -sv /bin/bash /bin/sh</span><br><span class="line"><span class="string">'/bin/sh'</span> -> <span class="string">'/bin/bash'</span></span><br><span class="line">+ <span class="built_in">echo</span> <span class="string">'auth required pam_wheel.so use_uid'</span></span><br><span class="line">+ chgrp root /etc/passwd</span><br><span class="line">+ chmod ug+rw /etc/passwd</span><br><span class="line">Removing intermediate container 52cff938c375</span><br><span class="line"> ---> afcf7dcb4c3f</span><br><span class="line">Step 6/15 : COPY <span class="variable">${spark_jars}</span> /opt/spark/jars</span><br><span class="line"> ---> 89385fbda28a</span><br><span class="line">Step 7/15 : COPY bin /opt/spark/bin</span><br><span class="line"> ---> aa3513116505</span><br><span class="line">Step 8/15 : COPY sbin /opt/spark/sbin</span><br><span class="line"> ---> 47bfb4c55137</span><br><span class="line">Step 9/15 : COPY <span class="variable">${img_path}</span>/spark/entrypoint.sh /opt/</span><br><span class="line"> ---> 4e1505375576</span><br><span class="line">Step 10/15 : COPY examples /opt/spark/examples</span><br><span class="line"> ---> 516d18d76b12</span><br><span class="line">Step 11/15 : COPY <span class="variable">${k8s_tests}</span> /opt/spark/tests</span><br><span class="line"> ---> 9f89a7c63732</span><br><span class="line">Step 12/15 : COPY data /opt/spark/data</span><br><span class="line"> ---> 4f630068f4e7</span><br><span class="line">Step 13/15 : ENV SPARK_HOME /opt/spark</span><br><span class="line"> ---> Running <span class="keyword">in</span> 5a3c23422bf6</span><br><span class="line">Removing intermediate container 5a3c23422bf6</span><br><span class="line"> ---> f2b96e5d55df</span><br><span class="line">Step 14/15 : WORKDIR /opt/spark/work-dir</span><br><span class="line"> ---> Running <span class="keyword">in</span> 7579f5960498</span><br><span class="line">Removing intermediate container 7579f5960498</span><br><span class="line"> ---> c2b2f4e804be</span><br><span class="line">Step 15/15 : ENTRYPOINT [ <span class="string">"/opt/entrypoint.sh"</span> ]</span><br><span class="line"> ---> Running <span class="keyword">in</span> 54c8dd48bf2c</span><br><span class="line">Removing intermediate container 54c8dd48bf2c</span><br><span class="line"> ---> 7d895d565b9d</span><br><span class="line">Successfully built 7d895d565b9d</span><br><span class="line">Successfully tagged 172.21.230.89/doublehappy/spark:v2.4.3</span><br><span class="line">Sending build context to Docker daemon 259.1MB</span><br><span class="line">Step 1/9 : ARG base_img</span><br><span class="line">Step 2/9 : FROM <span class="variable">$base_img</span></span><br><span class="line"> ---> 7d895d565b9d</span><br><span class="line">Step 3/9 : WORKDIR /</span><br><span class="line"> ---> Running <span class="keyword">in</span> 676ac68db202</span><br><span class="line">Removing intermediate container 676ac68db202</span><br><span class="line"> ---> 72e742d2ebe3</span><br><span class="line">Step 4/9 : RUN mkdir <span class="variable">${SPARK_HOME}</span>/python</span><br><span class="line"> ---> Running <span class="keyword">in</span> e7792928115a</span><br><span class="line">Removing intermediate container e7792928115a</span><br><span class="line"> ---> 00148d7dc2ff</span><br><span class="line">Step 5/9 : RUN apk add --no-cache python && apk add --no-cache python3 && python -m ensurepip && python3 -m ensurepip && rm -r /usr/lib/python*/ensurepip && pip install --upgrade pip setuptools && rm -r /root/.cache</span><br><span class="line"> ---> Running <span class="keyword">in</span> 959225826ce7</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz</span><br><span class="line">(1/3) Installing expat (2.2.8-r0)</span><br><span class="line">(2/3) Installing gdbm (1.13-r1)</span><br><span class="line">(3/3) Installing python2 (2.7.16-r2)</span><br><span class="line">Executing busybox-1.29.3-r10.trigger</span><br><span class="line">OK: 152 MiB <span class="keyword">in</span> 65 packages</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz</span><br><span class="line">(1/2) Installing xz-libs (5.2.4-r0)</span><br><span class="line">(2/2) Installing python3 (3.6.9-r2)</span><br><span class="line">Executing busybox-1.29.3-r10.trigger</span><br><span class="line">OK: 205 MiB <span class="keyword">in</span> 67 packages</span><br><span class="line">Looking <span class="keyword">in</span> links: /tmp/tmpxKrRmR</span><br><span class="line">Collecting setuptools</span><br><span class="line">Collecting pip</span><br><span class="line">Installing collected packages: setuptools, pip</span><br><span class="line">Successfully installed pip-18.1 setuptools-40.6.2</span><br><span class="line">Looking <span class="keyword">in</span> links: /tmp/tmp83v437xu</span><br><span class="line">Requirement already satisfied: setuptools <span class="keyword">in</span> /usr/lib/python3.6/site-packages (40.6.2)</span><br><span class="line">Requirement already satisfied: pip <span class="keyword">in</span> /usr/lib/python3.6/site-packages (18.1)</span><br><span class="line">Collecting pip</span><br><span class="line"> Downloading https://files.pythonhosted.org/packages/54/0c/d01aa759fdc501a58f431eb594a17495f15b88da142ce14b5845662c13f3/pip-20.0.2-py2.py3-none-any.whl (1.4MB)</span><br><span class="line">Collecting setuptools</span><br><span class="line"> Downloading https://files.pythonhosted.org/packages/f9/d3/955738b20d3832dfa3cd3d9b07e29a8162edb480bf988332f5e6e48ca444/setuptools-44.0.0-py2.py3-none-any.whl (583kB)</span><br><span class="line">Installing collected packages: pip, setuptools</span><br><span class="line"> Found existing installation: pip 18.1</span><br><span class="line"> Uninstalling pip-18.1:</span><br><span class="line"> Successfully uninstalled pip-18.1</span><br><span class="line"> Found existing installation: setuptools 40.6.2</span><br><span class="line"> Uninstalling setuptools-40.6.2:</span><br><span class="line"> Successfully uninstalled setuptools-40.6.2</span><br><span class="line">Successfully installed pip-20.0.2 setuptools-44.0.0</span><br><span class="line">Removing intermediate container 959225826ce7</span><br><span class="line"> ---> d803cfb93fb7</span><br><span class="line">Step 6/9 : COPY python/lib <span class="variable">${SPARK_HOME}</span>/python/lib</span><br><span class="line"> ---> a6ff2890f901</span><br><span class="line">Step 7/9 : ENV PYTHONPATH <span class="variable">${SPARK_HOME}</span>/python/lib/pyspark.zip:<span class="variable">${SPARK_HOME}</span>/python/lib/py4j-*.zip</span><br><span class="line"> ---> Running <span class="keyword">in</span> 239d8bd82251</span><br><span class="line">Removing intermediate container 239d8bd82251</span><br><span class="line"> ---> a3899eac9c76</span><br><span class="line">Step 8/9 : WORKDIR /opt/spark/work-dir</span><br><span class="line"> ---> Running <span class="keyword">in</span> afa1f7f2fba0</span><br><span class="line">Removing intermediate container afa1f7f2fba0</span><br><span class="line"> ---> 0924048e54c9</span><br><span class="line">Step 9/9 : ENTRYPOINT [ <span class="string">"/opt/entrypoint.sh"</span> ]</span><br><span class="line"> ---> Running <span class="keyword">in</span> 3775107781f5</span><br><span class="line">Removing intermediate container 3775107781f5</span><br><span class="line"> ---> 0807413bcdcd</span><br><span class="line">Successfully built 0807413bcdcd</span><br><span class="line">Successfully tagged 172.21.230.89/doublehappy/spark-py:v2.4.3</span><br><span class="line">Sending build context to Docker daemon 259.1MB</span><br><span class="line">Step 1/9 : ARG base_img</span><br><span class="line">Step 2/9 : FROM <span class="variable">$base_img</span></span><br><span class="line"> ---> 7d895d565b9d</span><br><span class="line">Step 3/9 : WORKDIR /</span><br><span class="line"> ---> Using cache</span><br><span class="line"> ---> 72e742d2ebe3</span><br><span class="line">Step 4/9 : RUN mkdir <span class="variable">${SPARK_HOME}</span>/R</span><br><span class="line"> ---> Running <span class="keyword">in</span> 7e96151ab7ee</span><br><span class="line">Removing intermediate container 7e96151ab7ee</span><br><span class="line"> ---> 8a1fde912f50</span><br><span class="line">Step 5/9 : RUN apk add --no-cache R R-dev</span><br><span class="line"> ---> Running <span class="keyword">in</span> f2a9cfb0ee09</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/main/x86_64/APKINDEX.tar.gz</span><br><span class="line">fetch http://dl-cdn.alpinelinux.org/alpine/v3.9/community/x86_64/APKINDEX.tar.gz</span><br><span class="line">(1/60) Installing R-mathlib (3.5.1-r1)</span><br><span class="line">(2/60) Installing libice (1.0.9-r3)</span><br><span class="line">(3/60) Installing libuuid (2.33-r0)</span><br><span class="line">(4/60) Installing libsm (1.2.2-r2)</span><br><span class="line">(5/60) Installing libxt (1.1.5-r2)</span><br><span class="line">(6/60) Installing libxmu (1.1.2-r1)</span><br><span class="line">(7/60) Installing expat (2.2.8-r0)</span><br><span class="line">(8/60) Installing fontconfig (2.13.1-r0)</span><br><span class="line">(9/60) Installing pixman (0.34.0-r6)</span><br><span class="line">(10/60) Installing cairo (1.16.0-r1)</span><br><span class="line">(11/60) Installing nghttp2-libs (1.35.1-r1)</span><br><span class="line">(12/60) Installing libssh2 (1.9.0-r1)</span><br><span class="line">(13/60) Installing libcurl (7.64.0-r3)</span><br><span class="line">(14/60) Installing libintl (0.19.8.1-r4)</span><br><span class="line">(15/60) Installing libblkid (2.33-r0)</span><br><span class="line">(16/60) Installing libmount (2.33-r0)</span><br><span class="line">(17/60) Installing pcre (8.42-r1)</span><br><span class="line">(18/60) Installing glib (2.58.1-r3)</span><br><span class="line">(19/60) Installing libgomp (8.3.0-r0)</span><br><span class="line">(20/60) Installing icu-libs (62.1-r0)</span><br><span class="line">(21/60) Installing xz-libs (5.2.4-r0)</span><br><span class="line">(22/60) Installing libquadmath (8.3.0-r0)</span><br><span class="line">(23/60) Installing libgfortran (8.3.0-r0)</span><br><span class="line">(24/60) Installing openblas (0.3.3-r2)</span><br><span class="line">(25/60) Installing libxft (2.3.2-r3)</span><br><span class="line">(26/60) Installing fribidi (1.0.5-r1)</span><br><span class="line">(27/60) Installing graphite2 (1.3.12-r1)</span><br><span class="line">(28/60) Installing harfbuzz (2.2.0-r0)</span><br><span class="line">(29/60) Installing pango (1.42.4-r1)</span><br><span class="line">(30/60) Installing tiff (4.0.10-r3)</span><br><span class="line">(31/60) Installing R (3.5.1-r1)</span><br><span class="line">Executing R-3.5.1-r1.post-install</span><br><span class="line">*</span><br><span class="line">* If you want to install R packages from CRAN that contains native extensions,</span><br><span class="line">* <span class="keyword">then</span> you must also install R-dev.</span><br><span class="line">*</span><br><span class="line">(32/60) Installing binutils (2.31.1-r2)</span><br><span class="line">(33/60) Installing gmp (6.1.2-r1)</span><br><span class="line">(34/60) Installing isl (0.18-r0)</span><br><span class="line">(35/60) Installing libatomic (8.3.0-r0)</span><br><span class="line">(36/60) Installing mpfr3 (3.1.5-r1)</span><br><span class="line">(37/60) Installing mpc1 (1.0.3-r1)</span><br><span class="line">(38/60) Installing gcc (8.3.0-r0)</span><br><span class="line">(39/60) Installing gfortran (8.3.0-r0)</span><br><span class="line">(40/60) Installing icu (62.1-r0)</span><br><span class="line">(41/60) Installing pkgconf (1.6.0-r0)</span><br><span class="line">(42/60) Installing icu-dev (62.1-r0)</span><br><span class="line">(43/60) Installing zlib-dev (1.2.11-r1)</span><br><span class="line">(44/60) Installing libpng-dev (1.6.37-r0)</span><br><span class="line">(45/60) Installing make (4.2.1-r2)</span><br><span class="line">(46/60) Installing openblas-ilp64 (0.3.3-r2)</span><br><span class="line">(47/60) Installing openblas-dev (0.3.3-r2)</span><br><span class="line">(48/60) Installing libpcre16 (8.42-r1)</span><br><span class="line">(49/60) Installing libpcre32 (8.42-r1)</span><br><span class="line">(50/60) Installing libpcrecpp (8.42-r1)</span><br><span class="line">(51/60) Installing pcre-dev (8.42-r1)</span><br><span class="line">(52/60) Installing libhistory (7.0.003-r1)</span><br><span class="line">(53/60) Installing readline-dev (7.0.003-r1)</span><br><span class="line">(54/60) Installing xz-dev (5.2.4-r0)</span><br><span class="line">(55/60) Installing bzip2-dev (1.0.6-r7)</span><br><span class="line">(56/60) Installing openssl-dev (1.1.1d-r2)</span><br><span class="line">(57/60) Installing libssh2-dev (1.9.0-r1)</span><br><span class="line">(58/60) Installing nghttp2-dev (1.35.1-r1)</span><br><span class="line">(59/60) Installing curl-dev (7.64.0-r3)</span><br><span class="line">(60/60) Installing R-dev (3.5.1-r1)</span><br><span class="line">Executing busybox-1.29.3-r10.trigger</span><br><span class="line">Executing glib-2.58.1-r3.trigger</span><br><span class="line">OK: 483 MiB <span class="keyword">in</span> 122 packages</span><br><span class="line">Removing intermediate container f2a9cfb0ee09</span><br><span class="line"> ---> 6ba3e96945f1</span><br><span class="line">Step 6/9 : COPY R <span class="variable">${SPARK_HOME}</span>/R</span><br><span class="line"> ---> 53c866b24bc7</span><br><span class="line">Step 7/9 : ENV R_HOME /usr/lib/R</span><br><span class="line"> ---> Running <span class="keyword">in</span> 353fcf7a3195</span><br><span class="line">Removing intermediate container 353fcf7a3195</span><br><span class="line"> ---> 15f73ab74099</span><br><span class="line">Step 8/9 : WORKDIR /opt/spark/work-dir</span><br><span class="line"> ---> Running <span class="keyword">in</span> 6f8f14f50dd1</span><br><span class="line">Removing intermediate container 6f8f14f50dd1</span><br><span class="line"> ---> f9ba7e7be95f</span><br><span class="line">Step 9/9 : ENTRYPOINT [ <span class="string">"/opt/entrypoint.sh"</span> ]</span><br><span class="line"> ---> Running <span class="keyword">in</span> b45e8b5b55ca</span><br><span class="line">Removing intermediate container b45e8b5b55ca</span><br><span class="line"> ---> e77ac91a3898</span><br><span class="line">Successfully built e77ac91a3898</span><br><span class="line">Successfully tagged 172.21.230.89/doublehappy/spark-r:v2.4.3</span><br><span class="line">[root@container01 spark]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">查看一下：</span><br><span class="line"></span><br><span class="line">[root@container01 ~]<span class="comment"># docker images</span></span><br><span class="line">REPOSITORY TAG IMAGE ID CREATED SIZE</span><br><span class="line"><none> <none> 8a1fde912f50 12 seconds ago 371MB</span><br><span class="line">172.21.230.89/doublehappy/spark-py v2.4.3 0807413bcdcd 15 seconds ago 462MB</span><br><span class="line">172.21.230.89/doublehappy/spark v2.4.3 7d895d565b9d 36 seconds ago 371MB</span><br><span class="line">mysql doublehappy5.7 5c68aaadf900 4 days ago 434MB</span><br><span class="line">debian stretch-slim 5174edbf8c39 9 days ago 55.3MB</span><br><span class="line">goharbor/chartmuseum-photon v0.9.0-v1.10.0 c8c98d2ac22b 2 months ago 128MB</span><br><span class="line">goharbor/harbor-migrator v1.10.0 9dd0c9228c79 2 months ago 362MB</span><br><span class="line">goharbor/redis-photon v1.10.0 6df66e5c1ca7 2 months ago 111MB</span><br><span class="line">goharbor/clair-adapter-photon v1.0.1-v1.10.0 4bdc58f374c2 2 months ago 61.6MB</span><br><span class="line">goharbor/clair-photon v2.1.1-v1.10.0 435d235c5000 2 months ago 171MB</span><br><span class="line">goharbor/notary-server-photon v0.6.1-v1.10.0 c7934c32e873 2 months ago 143MB</span><br><span class="line">goharbor/notary-signer-photon v0.6.1-v1.10.0 bb1a762155ab 2 months ago 140MB</span><br><span class="line">goharbor/harbor-registryctl v1.10.0 c550280445e6 2 months ago 104MB</span><br><span class="line">goharbor/registry-photon v2.7.1-patch-2819-2553-v1.10.0 2115e08fa399 2 months ago 86.5MB</span><br><span class="line">goharbor/nginx-photon v1.10.0 f7ed614c3abc 2 months ago 44MB</span><br><span class="line">goharbor/harbor-log v1.10.0 fb15f6772e9a 2 months ago 82.3MB</span><br><span class="line">goharbor/harbor-jobservice v1.10.0 d6d4f2b125f6 2 months ago 142MB</span><br><span class="line">goharbor/harbor-core v1.10.0 f3a3065b3af2 2 months ago 128MB</span><br><span class="line">goharbor/harbor-portal v1.10.0 fbaeb1fdacad 2 months ago 52.1MB</span><br><span class="line">goharbor/harbor-db v1.10.0 634404a417cf 2 months ago 148MB</span><br><span class="line">goharbor/prepare v1.10.0 927062458494 2 months ago 149MB</span><br><span class="line">openjdk 8-alpine a3562aa0b991 9 months ago 105MB</span><br><span class="line">k8s.gcr.io/kube-controller-manager v1.13.2 b9027a78d94c 13 months ago 146MB</span><br><span class="line">hackeruncle/kube-controller-manager v1.13.2 b9027a78d94c 13 months ago 146MB</span><br><span class="line">hackeruncle/kube-proxy v1.13.2 01cfa56edcfc 13 months ago 80.3MB</span><br><span class="line">k8s.gcr.io/kube-proxy v1.13.2 01cfa56edcfc 13 months ago 80.3MB</span><br><span class="line">hackeruncle/kube-apiserver v1.13.2 177db4b8e93a 13 months ago 181MB</span><br><span class="line">k8s.gcr.io/kube-apiserver v1.13.2 177db4b8e93a 13 months ago 181MB</span><br><span class="line">hackeruncle/kube-scheduler v1.13.2 3193be46e0b3 13 months ago 79.6MB</span><br><span class="line">k8s.gcr.io/kube-scheduler v1.13.2 3193be46e0b3 13 months ago 79.6MB</span><br><span class="line">registry.cn-beijing.aliyuncs.com/minminmsn/kubernetes-dashboard v1.10.1 2b631cbc40bc 13 months ago 122MB</span><br><span class="line">172.21.230.89/doublehappy/hello-world v1 fce289e99eb9 13 months ago 1.84kB</span><br><span class="line">container01/doublehappy/hello-world v2 fce289e99eb9 13 months ago 1.84kB</span><br><span class="line">k8s.gcr.io/coredns 1.2.6 f59dcacceff4 15 months ago 40MB</span><br><span class="line">hackeruncle/coredns 1.2.6 f59dcacceff4 15 months ago 40MB</span><br><span class="line">hackeruncle/etcd 3.2.24 3cab8e1b9802 16 months ago 220MB</span><br><span class="line">k8s.gcr.io/etcd 3.2.24 3cab8e1b9802 16 months ago 220MB</span><br><span class="line">quay.io/coreos/flannel v0.10.0-amd64 f0fad859c909 2 years ago 44.6MB</span><br><span class="line">hackeruncle/pause 3.1 da86e6ba6ca1 2 years ago 742kB</span><br><span class="line">k8s.gcr.io/pause 3.1 da86e6ba6ca1 2 years ago 742kB</span><br><span class="line">172.21.230.89/doublehappy/mysql-slave v1.0 b152e49ead77 3 years ago 426MB</span><br><span class="line">hackeruncle/mysql-slave v1.0 b152e49ead77 3 years ago 426MB</span><br><span class="line">172.21.230.89/doublehappy/mysql-master v1.0 b73ad03215d5 3 years ago 426MB</span><br><span class="line">hackeruncle/mysql-master v1.0 b73ad03215d5 3 years ago 426MB</span><br><span class="line">[root@container01 ~]<span class="comment"># docker images</span></span><br><span class="line">REPOSITORY TAG IMAGE ID CREATED SIZE</span><br><span class="line">172.21.230.89/doublehappy/spark-r v2.4.3 e77ac91a3898 About a minute ago 757MB</span><br><span class="line">172.21.230.89/doublehappy/spark-py v2.4.3 0807413bcdcd About a minute ago 462MB</span><br><span class="line">172.21.230.89/doublehappy/spark v2.4.3 7d895d565b9d About a minute ago 371MB</span><br><span class="line">mysql doublehappy5.7 5c68aaadf900 4 days ago 434MB</span><br><span class="line">debian stretch-slim 5174edbf8c39 9 days ago 55.3MB</span><br><span class="line">goharbor/chartmuseum-photon v0.9.0-v1.10.0 c8c98d2ac22b 2 months ago 128MB</span><br><span class="line">goharbor/harbor-migrator v1.10.0 9dd0c9228c79 2 months ago 362MB</span><br><span class="line">goharbor/redis-photon v1.10.0 6df66e5c1ca7 2 months ago 111MB</span><br><span class="line">goharbor/clair-adapter-photon v1.0.1-v1.10.0 4bdc58f374c2 2 months ago 61.6MB</span><br><span class="line">goharbor/clair-photon v2.1.1-v1.10.0 435d235c5000 2 months ago 171MB</span><br><span class="line">goharbor/notary-server-photon v0.6.1-v1.10.0 c7934c32e873 2 months ago 143MB</span><br><span class="line">goharbor/notary-signer-photon v0.6.1-v1.10.0 bb1a762155ab 2 months ago 140MB</span><br><span class="line">goharbor/harbor-registryctl v1.10.0 c550280445e6 2 months ago 104MB</span><br><span class="line">goharbor/registry-photon v2.7.1-patch-2819-2553-v1.10.0 2115e08fa399 2 months ago 86.5MB</span><br><span class="line">goharbor/nginx-photon v1.10.0 f7ed614c3abc 2 months ago 44MB</span><br><span class="line">goharbor/harbor-log v1.10.0 fb15f6772e9a 2 months ago 82.3MB</span><br><span class="line">goharbor/harbor-jobservice v1.10.0 d6d4f2b125f6 2 months ago 142MB</span><br><span class="line">goharbor/harbor-core v1.10.0 f3a3065b3af2 2 months ago 128MB</span><br><span class="line">goharbor/harbor-portal v1.10.0 fbaeb1fdacad 2 months ago 52.1MB</span><br><span class="line">goharbor/harbor-db v1.10.0 634404a417cf 2 months ago 148MB</span><br><span class="line">goharbor/prepare v1.10.0 927062458494 2 months ago 149MB</span><br><span class="line">openjdk 8-alpine a3562aa0b991 9 months ago 105MB</span><br><span class="line">hackeruncle/kube-apiserver v1.13.2 177db4b8e93a 13 months ago 181MB</span><br><span class="line">k8s.gcr.io/kube-apiserver v1.13.2 177db4b8e93a 13 months ago 181MB</span><br><span class="line">hackeruncle/kube-proxy v1.13.2 01cfa56edcfc 13 months ago 80.3MB</span><br><span class="line">k8s.gcr.io/kube-proxy v1.13.2 01cfa56edcfc 13 months ago 80.3MB</span><br><span class="line">hackeruncle/kube-controller-manager v1.13.2 b9027a78d94c 13 months ago 146MB</span><br><span class="line">k8s.gcr.io/kube-controller-manager v1.13.2 b9027a78d94c 13 months ago 146MB</span><br><span class="line">hackeruncle/kube-scheduler v1.13.2 3193be46e0b3 13 months ago 79.6MB</span><br><span class="line">k8s.gcr.io/kube-scheduler v1.13.2 3193be46e0b3 13 months ago 79.6MB</span><br><span class="line">registry.cn-beijing.aliyuncs.com/minminmsn/kubernetes-dashboard v1.10.1 2b631cbc40bc 13 months ago 122MB</span><br><span class="line">172.21.230.89/doublehappy/hello-world v1 fce289e99eb9 13 months ago 1.84kB</span><br><span class="line">container01/doublehappy/hello-world v2 fce289e99eb9 13 months ago 1.84kB</span><br><span class="line">k8s.gcr.io/coredns 1.2.6 f59dcacceff4 15 months ago 40MB</span><br><span class="line">hackeruncle/coredns 1.2.6 f59dcacceff4 15 months ago 40MB</span><br><span class="line">k8s.gcr.io/etcd 3.2.24 3cab8e1b9802 16 months ago 220MB</span><br><span class="line">hackeruncle/etcd 3.2.24 3cab8e1b9802 16 months ago 220MB</span><br><span class="line">quay.io/coreos/flannel v0.10.0-amd64 f0fad859c909 2 years ago 44.6MB</span><br><span class="line">hackeruncle/pause 3.1 da86e6ba6ca1 2 years ago 742kB</span><br><span class="line">k8s.gcr.io/pause 3.1 da86e6ba6ca1 2 years ago 742kB</span><br><span class="line">172.21.230.89/doublehappy/mysql-slave v1.0 b152e49ead77 3 years ago 426MB</span><br><span class="line">hackeruncle/mysql-slave v1.0 b152e49ead77 3 years ago 426MB</span><br><span class="line">172.21.230.89/doublehappy/mysql-master v1.0 b73ad03215d5 3 years ago 426MB</span><br><span class="line">hackeruncle/mysql-master v1.0 b73ad03215d5 3 years ago 426MB</span><br><span class="line">[root@container01 ~]<span class="comment"># </span></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">push：到私服</span><br><span class="line"></span><br><span class="line">[root@container01 spark]<span class="comment"># ./bin/docker-image-tool.sh -r 172.21.230.89/doublehappy -t v2.4.3 push</span></span><br><span class="line">The push refers to repository [172.21.230.89/doublehappy/spark]</span><br><span class="line">64f8d2d5dc6b: Pushed </span><br><span class="line">8bad673cc19a: Pushed </span><br><span class="line">37a326771171: Pushed </span><br><span class="line">6334bffbbe5f: Pushed </span><br><span class="line">e47081996b74: Pushed </span><br><span class="line">897cdaf31bf7: Pushed </span><br><span class="line">261b1b5eb382: Pushed </span><br><span class="line">5add1f9f7507: Pushed </span><br><span class="line">ceaf9e1ebef5: Pushed </span><br><span class="line">9b9b7f3d56a0: Pushed </span><br><span class="line">f1b5933fe4b5: Pushed </span><br><span class="line">v2.4.3: digest: sha256:c8f97b50e7c325a9ad342afa7e796ae594f802306fae82b82d9622a462b32fa3 size: 2624</span><br><span class="line">The push refers to repository [172.21.230.89/doublehappy/spark-py]</span><br><span class="line">b161c5b08e39: Pushed </span><br><span class="line">abd349f5b3b6: Pushed </span><br><span class="line">e48b59a107cb: Pushed </span><br><span class="line">64f8d2d5dc6b: Mounted from doublehappy/spark </span><br><span class="line">8bad673cc19a: Mounted from doublehappy/spark </span><br><span class="line">37a326771171: Mounted from doublehappy/spark </span><br><span class="line">6334bffbbe5f: Mounted from doublehappy/spark </span><br><span class="line">e47081996b74: Mounted from doublehappy/spark </span><br><span class="line">897cdaf31bf7: Mounted from doublehappy/spark </span><br><span class="line">261b1b5eb382: Mounted from doublehappy/spark </span><br><span class="line">5add1f9f7507: Mounted from doublehappy/spark </span><br><span class="line">ceaf9e1ebef5: Mounted from doublehappy/spark </span><br><span class="line">9b9b7f3d56a0: Mounted from doublehappy/spark </span><br><span class="line">f1b5933fe4b5: Mounted from doublehappy/spark </span><br><span class="line">v2.4.3: digest: sha256:d466a3c0ac638e6c14048aabf6d48307f9c1ee691a8817ef5a40aeb24e420824 size: 3253</span><br><span class="line">The push refers to repository [172.21.230.89/doublehappy/spark-r]</span><br><span class="line">9f1cc9476665: Pushed </span><br><span class="line">4ef658c87c9d: Pushed </span><br><span class="line">13685a4568e4: Pushed </span><br><span class="line">64f8d2d5dc6b: Mounted from doublehappy/spark-py </span><br><span class="line">8bad673cc19a: Mounted from doublehappy/spark-py </span><br><span class="line">37a326771171: Mounted from doublehappy/spark-py </span><br><span class="line">6334bffbbe5f: Mounted from doublehappy/spark-py </span><br><span class="line">e47081996b74: Mounted from doublehappy/spark-py </span><br><span class="line">897cdaf31bf7: Mounted from doublehappy/spark-py </span><br><span class="line">261b1b5eb382: Mounted from doublehappy/spark-py </span><br><span class="line">5add1f9f7507: Mounted from doublehappy/spark-py </span><br><span class="line">ceaf9e1ebef5: Mounted from doublehappy/spark-py </span><br><span class="line">9b9b7f3d56a0: Mounted from doublehappy/spark-py </span><br><span class="line">f1b5933fe4b5: Mounted from doublehappy/spark-py </span><br><span class="line">v2.4.3: digest: sha256:9aad314aae767666e0fa6605753ab95d758b7286ef63fca6e7bab5732cbe4030 size: 3255</span><br><span class="line">[root@container01 spark]<span class="comment">#</span></span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200211154658637.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTU1NzUxMg==,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br><span class="line">245</span><br><span class="line">246</span><br><span class="line">247</span><br><span class="line">248</span><br><span class="line">249</span><br><span class="line">250</span><br><span class="line">251</span><br><span class="line">252</span><br><span class="line">253</span><br><span class="line">254</span><br><span class="line">255</span><br><span class="line">256</span><br><span class="line">257</span><br><span class="line">258</span><br><span class="line">259</span><br><span class="line">260</span><br><span class="line">261</span><br><span class="line">262</span><br><span class="line">263</span><br><span class="line">264</span><br><span class="line">265</span><br><span class="line">266</span><br><span class="line">267</span><br><span class="line">268</span><br><span class="line">269</span><br><span class="line">270</span><br><span class="line">271</span><br><span class="line">272</span><br><span class="line">273</span><br><span class="line">274</span><br><span class="line">275</span><br><span class="line">276</span><br><span class="line">277</span><br><span class="line">278</span><br><span class="line">279</span><br><span class="line">280</span><br></pre></td><td class="code"><pre><span class="line">3.创建secret</span><br><span class="line">kubectl create secret docker-registry sparkkey \</span><br><span class="line">--namespace=default \</span><br><span class="line">--docker-server=172.21.230.89 \</span><br><span class="line">--docker-username=admin \</span><br><span class="line">--docker-password=Harbor12345 </span><br><span class="line"></span><br><span class="line">[root@container01 spark]# kubectl create secret docker-registry sparkkey \</span><br><span class="line">> --namespace=default \</span><br><span class="line">> --docker-server=172.21.230.89 \</span><br><span class="line">> --docker-username=admin \</span><br><span class="line">> --docker-password=Harbor12345 </span><br><span class="line">secret/sparkkey created</span><br><span class="line">[root@container01 spark]# </span><br><span class="line"></span><br><span class="line">4.创建serviceAccount及 绑定clusterrolebinding</span><br><span class="line">kubectl create serviceaccount spark</span><br><span class="line"></span><br><span class="line">kubectl create clusterrolebinding spark-role --clusterrole=edit \</span><br><span class="line">--serviceaccount=default:spark --namespace=default</span><br><span class="line"></span><br><span class="line">[root@container01 spark]# kubectl create serviceaccount spark</span><br><span class="line">serviceaccount/spark created</span><br><span class="line">[root@container01 spark]# kubectl create clusterrolebinding spark-role --clusterrole=edit \</span><br><span class="line">> --serviceaccount=default:spark --namespace=default</span><br><span class="line">clusterrolebinding.rbac.authorization.k8s.io/spark-role created</span><br><span class="line">[root@container01 spark]# </span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">5. 提交之前 查看 k8s port :k8s-apiserver-port (看spark官网)</span><br><span class="line"></span><br><span class="line">[root@container01 spark]# kubectl cluster-info</span><br><span class="line">Kubernetes master is running at https://172.21.230.89:6443</span><br><span class="line">KubeDNS is running at https://172.21.230.89:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy</span><br><span class="line"></span><br><span class="line">To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.</span><br><span class="line">[root@container01 spark]# </span><br><span class="line"></span><br><span class="line">即：https://172.21.230.89:6443</span><br><span class="line"></span><br><span class="line">bin/spark-submit \</span><br><span class="line">--master k8s://https://172.21.230.89:6443 \</span><br><span class="line">--deploy-mode cluster \</span><br><span class="line">--name spark-pi \</span><br><span class="line">--class org.apache.spark.examples.SparkPi \</span><br><span class="line">--conf spark.executor.instances=2 \</span><br><span class="line">--conf spark.kubernetes.container.image.pullSecrets=sparkkey \</span><br><span class="line">--conf spark.kubernetes.container.image=172.21.230.89/doublehappy/spark:v2.4.3 \</span><br><span class="line">--conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \</span><br><span class="line">--conf spark.kubernetes.driver.pod.name=spark-pi-driver \</span><br><span class="line">local:///opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">这些参数spark官网都有 下面又截图</span><br><span class="line"></span><br><span class="line">[root@container01 spark]# bin/spark-submit \</span><br><span class="line">> --master k8s://https://172.21.230.89:6443 \</span><br><span class="line">> --deploy-mode cluster \</span><br><span class="line">> --name spark-pi \</span><br><span class="line">> --class org.apache.spark.examples.SparkPi \</span><br><span class="line">> --conf spark.executor.instances=2 \</span><br><span class="line">> --conf spark.kubernetes.container.image.pullSecrets=sparkkey \</span><br><span class="line">> --conf spark.kubernetes.container.image=172.21.230.89/doublehappy/spark:v2.4.3 \</span><br><span class="line">> --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \</span><br><span class="line">> --conf spark.kubernetes.driver.pod.name=spark-pi-driver \</span><br><span class="line">> local:///opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar</span><br><span class="line">log4j:WARN No appenders could be found for logger (io.fabric8.kubernetes.client.Config).</span><br><span class="line">log4j:WARN Please initialize the log4j system properly.</span><br><span class="line">log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.</span><br><span class="line">Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties</span><br><span class="line">20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: </span><br><span class="line"> pod name: spark-pi-driver</span><br><span class="line"> namespace: default</span><br><span class="line"> labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver</span><br><span class="line"> pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece</span><br><span class="line"> creation time: 2020-02-11T08:16:48Z</span><br><span class="line"> service account name: spark</span><br><span class="line"> volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn</span><br><span class="line"> node name: N/A</span><br><span class="line"> start time: N/A</span><br><span class="line"> container images: N/A</span><br><span class="line"> phase: Pending</span><br><span class="line"> status: []</span><br><span class="line">20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: </span><br><span class="line"> pod name: spark-pi-driver</span><br><span class="line"> namespace: default</span><br><span class="line"> labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver</span><br><span class="line"> pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece</span><br><span class="line"> creation time: 2020-02-11T08:16:48Z</span><br><span class="line"> service account name: spark</span><br><span class="line"> volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn</span><br><span class="line"> node name: container02</span><br><span class="line"> start time: N/A</span><br><span class="line"> container images: N/A</span><br><span class="line"> phase: Pending</span><br><span class="line"> status: []</span><br><span class="line">20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: </span><br><span class="line"> pod name: spark-pi-driver</span><br><span class="line"> namespace: default</span><br><span class="line"> labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver</span><br><span class="line"> pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece</span><br><span class="line"> creation time: 2020-02-11T08:16:48Z</span><br><span class="line"> service account name: spark</span><br><span class="line"> volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn</span><br><span class="line"> node name: container02</span><br><span class="line"> start time: 2020-02-11T08:16:49Z</span><br><span class="line"> container images: 172.21.230.89/doublehappy/spark:v2.4.3</span><br><span class="line"> phase: Pending</span><br><span class="line"> status: [ContainerStatus(containerID=null, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=false, restartCount=0, state=ContainerState(running=null, terminated=null, waiting=ContainerStateWaiting(message=null, reason=ContainerCreating, additionalProperties={}), additionalProperties={}), additionalProperties={})]</span><br><span class="line">20/02/11 16:16:49 INFO Client: Waiting for application spark-pi to finish...</span><br><span class="line">20/02/11 16:16:59 INFO LoggingPodStatusWatcherImpl: State changed, new state: </span><br><span class="line"> pod name: spark-pi-driver</span><br><span class="line"> namespace: default</span><br><span class="line"> labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver</span><br><span class="line"> pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece</span><br><span class="line"> creation time: 2020-02-11T08:16:48Z</span><br><span class="line"> service account name: spark</span><br><span class="line"> volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn</span><br><span class="line"> node name: container02</span><br><span class="line"> start time: 2020-02-11T08:16:49Z</span><br><span class="line"> container images: 172.21.230.89/doublehappy/spark:v2.4.3</span><br><span class="line"> phase: Running</span><br><span class="line"> status: [ContainerStatus(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=docker-pullable://172.21.230.89/doublehappy/spark@sha256:c8f97b50e7c325a9ad342afa7e796ae594f802306fae82b82d9622a462b32fa3, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=true, restartCount=0, state=ContainerState(running=ContainerStateRunning(startedAt=2020-02-11T08:16:59Z, additionalProperties={}), terminated=null, waiting=null, additionalProperties={}), additionalProperties={})]</span><br><span class="line">20/02/11 16:17:34 INFO LoggingPodStatusWatcherImpl: State changed, new state: </span><br><span class="line"> pod name: spark-pi-driver</span><br><span class="line"> namespace: default</span><br><span class="line"> labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver</span><br><span class="line"> pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece</span><br><span class="line"> creation time: 2020-02-11T08:16:48Z</span><br><span class="line"> service account name: spark</span><br><span class="line"> volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn</span><br><span class="line"> node name: container02</span><br><span class="line"> start time: 2020-02-11T08:16:49Z</span><br><span class="line"> container images: 172.21.230.89/doublehappy/spark:v2.4.3</span><br><span class="line"> phase: Succeeded</span><br><span class="line"> status: [ContainerStatus(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=docker-pullable://172.21.230.89/doublehappy/spark@sha256:c8f97b50e7c325a9ad342afa7e796ae594f802306fae82b82d9622a462b32fa3, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=false, restartCount=0, state=ContainerState(running=null, terminated=ContainerStateTerminated(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, exitCode=0, finishedAt=2020-02-11T08:17:34Z, message=null, reason=Completed, signal=null, startedAt=2020-02-11T08:16:59Z, additionalProperties={}), waiting=null, additionalProperties={}), additionalProperties={})]</span><br><span class="line">20/02/11 16:17:34 INFO LoggingPodStatusWatcherImpl: Container final statuses:</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> Container name: spark-kubernetes-driver</span><br><span class="line"> Container image: 172.21.230.89/doublehappy/spark:v2.4.3</span><br><span class="line"> Container state: Terminated</span><br><span class="line"> Exit code: 0</span><br><span class="line">20/02/11 16:17:34 INFO Client: Application spark-pi finished.</span><br><span class="line">20/02/11 16:17:34 INFO ShutdownHookManager: Shutdown hook called</span><br><span class="line">20/02/11 16:17:34 INFO ShutdownHookManager: Deleting directory /tmp/spark-41d1f84f-677e-4476-9cdc-a277ebe96dfa</span><br><span class="line">[root@container01 spark]# </span><br><span class="line"></span><br><span class="line">查看一下：</span><br><span class="line">[root@container01 ~]# kubectl get all</span><br><span class="line">NAME READY STATUS RESTARTS AGE</span><br><span class="line">pod/spark-pi-driver 0/1 Completed 0 92s</span><br><span class="line"></span><br><span class="line">NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE</span><br><span class="line">service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 26h</span><br><span class="line">service/spark-pi-1581409007077-driver-svc ClusterIP None <none> 7078/TCP,7079/TCP 91s</span><br><span class="line">[root@container01 ~]#</span><br><span class="line"></span><br><span class="line">查看spark log：</span><br><span class="line"></span><br><span class="line">[root@container01 ~]# kubectl logs -f spark-pi-driver</span><br><span class="line">++ id -u</span><br><span class="line">+ myuid=0</span><br><span class="line">++ id -g</span><br><span class="line">+ mygid=0</span><br><span class="line">+ set +e</span><br><span class="line">++ getent passwd 0</span><br><span class="line">+ uidentry=root:x:0:0:root:/root:/bin/ash</span><br><span class="line">+ set -e</span><br><span class="line">+ '[' -z root:x:0:0:root:/root:/bin/ash ']'</span><br><span class="line">+ SPARK_K8S_CMD=driver</span><br><span class="line">+ case "$SPARK_K8S_CMD" in</span><br><span class="line">+ shift 1</span><br><span class="line">+ SPARK_CLASSPATH=':/opt/spark/jars/*'</span><br><span class="line">+ grep SPARK_JAVA_OPT_</span><br><span class="line">+ sort -t_ -k4 -n</span><br><span class="line">+ sed 's/[^=]*=\(.*\)/\1/g'</span><br><span class="line">+ env</span><br><span class="line">+ readarray -t SPARK_EXECUTOR_JAVA_OPTS</span><br><span class="line">+ '[' -n '' ']'</span><br><span class="line">+ '[' -n '' ']'</span><br><span class="line">+ PYSPARK_ARGS=</span><br><span class="line">+ '[' -n '' ']'</span><br><span class="line">+ R_ARGS=</span><br><span class="line">+ '[' -n '' ']'</span><br><span class="line">+ '[' '' == 2 ']'</span><br><span class="line">+ '[' '' == 3 ']'</span><br><span class="line">+ case "$SPARK_K8S_CMD" in</span><br><span class="line">+ CMD=("$SPARK_HOME/bin/spark-submit" --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" --deploy-mode client "$@")</span><br><span class="line">+ exec /sbin/tini -s -- /opt/spark/bin/spark-submit --conf spark.driver.bindAddress=10.244.1.5 --deploy-mode client --properties-file /opt/spark/conf/spark.properties --class org.apache.spark.examples.SparkPi spark-internal</span><br><span class="line">20/02/11 08:17:00 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties</span><br><span class="line">20/02/11 08:17:01 INFO SparkContext: Running Spark version 2.4.3</span><br><span class="line">20/02/11 08:17:01 INFO SparkContext: Submitted application: Spark Pi</span><br><span class="line">20/02/11 08:17:01 INFO SecurityManager: Changing view acls to: root</span><br><span class="line">20/02/11 08:17:01 INFO SecurityManager: Changing modify acls to: root</span><br><span class="line">20/02/11 08:17:01 INFO SecurityManager: Changing view acls groups to: </span><br><span class="line">20/02/11 08:17:01 INFO SecurityManager: Changing modify acls groups to: </span><br><span class="line">20/02/11 08:17:01 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set()</span><br><span class="line">20/02/11 08:17:01 INFO Utils: Successfully started service 'sparkDriver' on port 7078.</span><br><span class="line">20/02/11 08:17:01 INFO SparkEnv: Registering MapOutputTracker</span><br><span class="line">20/02/11 08:17:01 INFO SparkEnv: Registering BlockManagerMaster</span><br><span class="line">20/02/11 08:17:01 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information</span><br><span class="line">20/02/11 08:17:01 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up</span><br><span class="line">20/02/11 08:17:01 INFO DiskBlockManager: Created local directory at /var/data/spark-6753da05-ab60-4e01-bafd-70e0141b47f2/blockmgr-f9c8a8f5-32c4-42b4-bb3b-3ec9686c04f5</span><br><span class="line">20/02/11 08:17:01 INFO MemoryStore: MemoryStore started with capacity 413.9 MB</span><br><span class="line">20/02/11 08:17:01 INFO SparkEnv: Registering OutputCommitCoordinator</span><br><span class="line">20/02/11 08:17:01 INFO Utils: Successfully started service 'SparkUI' on port 4040.</span><br><span class="line">20/02/11 08:17:01 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://spark-pi-1581409007077-driver-svc.default.svc:4040</span><br><span class="line">20/02/11 08:17:01 INFO SparkContext: Added JAR file:///opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar at spark://spark-pi-1581409007077-driver-svc.default.svc:7078/jars/spark-examples_2.11-2.4.3.jar with timestamp 1581409021888</span><br><span class="line">20/02/11 08:17:03 INFO ExecutorPodsAllocator: Going to request 2 executors from Kubernetes.</span><br><span class="line">20/02/11 08:17:03 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 7079.</span><br><span class="line">20/02/11 08:17:03 INFO NettyBlockTransferService: Server created on spark-pi-1581409007077-driver-svc.default.svc:7079</span><br><span class="line">20/02/11 08:17:03 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy</span><br><span class="line">20/02/11 08:17:03 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None)</span><br><span class="line">20/02/11 08:17:03 INFO BlockManagerMasterEndpoint: Registering block manager spark-pi-1581409007077-driver-svc.default.svc:7079 with 413.9 MB RAM, BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None)</span><br><span class="line">20/02/11 08:17:03 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None)</span><br><span class="line">20/02/11 08:17:03 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None)</span><br><span class="line">20/02/11 08:17:14 INFO KubernetesClusterSchedulerBackend$KubernetesDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.244.2.4:56814) with ID 1</span><br><span class="line">20/02/11 08:17:14 INFO BlockManagerMasterEndpoint: Registering block manager 10.244.2.4:37098 with 413.9 MB RAM, BlockManagerId(1, 10.244.2.4, 37098, None)</span><br><span class="line">20/02/11 08:17:33 INFO KubernetesClusterSchedulerBackend: SchedulerBackend is ready for scheduling beginning after waiting maxRegisteredResourcesWaitingTime: 30000(ms)</span><br><span class="line">20/02/11 08:17:33 INFO SparkContext: Starting job: reduce at SparkPi.scala:38</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Got job 0 (reduce at SparkPi.scala:38) with 2 output partitions</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Final stage: ResultStage 0 (reduce at SparkPi.scala:38)</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Parents of final stage: List()</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Missing parents: List()</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34), which has no missing parents</span><br><span class="line">20/02/11 08:17:33 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 1936.0 B, free 413.9 MB)</span><br><span class="line">20/02/11 08:17:33 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1256.0 B, free 413.9 MB)</span><br><span class="line">20/02/11 08:17:33 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on spark-pi-1581409007077-driver-svc.default.svc:7079 (size: 1256.0 B, free: 413.9 MB)</span><br><span class="line">20/02/11 08:17:33 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1161</span><br><span class="line">20/02/11 08:17:33 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34) (first 15 tasks are for partitions Vector(0, 1))</span><br><span class="line">20/02/11 08:17:33 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks</span><br><span class="line">20/02/11 08:17:33 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 10.244.2.4, executor 1, partition 0, PROCESS_LOCAL, 7885 bytes)</span><br><span class="line">20/02/11 08:17:34 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 10.244.2.4:37098 (size: 1256.0 B, free: 413.9 MB)</span><br><span class="line">20/02/11 08:17:34 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 10.244.2.4, executor 1, partition 1, PROCESS_LOCAL, 7885 bytes)</span><br><span class="line">20/02/11 08:17:34 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 534 ms on 10.244.2.4 (executor 1) (1/2)</span><br><span class="line">20/02/11 08:17:34 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 27 ms on 10.244.2.4 (executor 1) (2/2)</span><br><span class="line">20/02/11 08:17:34 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool </span><br><span class="line">20/02/11 08:17:34 INFO DAGScheduler: ResultStage 0 (reduce at SparkPi.scala:38) finished in 0.814 s</span><br><span class="line">20/02/11 08:17:34 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 0.868531 s</span><br><span class="line">Pi is roughly 3.139475697378487</span><br><span class="line">20/02/11 08:17:34 INFO SparkUI: Stopped Spark web UI at http://spark-pi-1581409007077-driver-svc.default.svc:4040</span><br><span class="line">20/02/11 08:17:34 INFO KubernetesClusterSchedulerBackend: Shutting down all executors</span><br><span class="line">20/02/11 08:17:34 INFO KubernetesClusterSchedulerBackend$KubernetesDriverEndpoint: Asking each executor to shut down</span><br><span class="line">20/02/11 08:17:34 WARN ExecutorPodsWatchSnapshotSource: Kubernetes client has been closed (this is expected if the application is shutting down.)</span><br><span class="line">20/02/11 08:17:34 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!</span><br><span class="line">20/02/11 08:17:34 INFO MemoryStore: MemoryStore cleared</span><br><span class="line">20/02/11 08:17:34 INFO BlockManager: BlockManager stopped</span><br><span class="line">20/02/11 08:17:34 INFO BlockManagerMaster: BlockManagerMaster stopped</span><br><span class="line">20/02/11 08:17:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!</span><br><span class="line">20/02/11 08:17:34 INFO SparkContext: Successfully stopped SparkContext</span><br><span class="line">20/02/11 08:17:34 INFO ShutdownHookManager: Shutdown hook called</span><br><span class="line">20/02/11 08:17:34 INFO ShutdownHookManager: Deleting directory /var/data/spark-6753da05-ab60-4e01-bafd-70e0141b47f2/spark-67555fc1-109a-46a0-bff7-1c14b972cfe4</span><br><span class="line">20/02/11 08:17:34 INFO ShutdownHookManager: Deleting directory /tmp/spark-7f09677c-5829-434e-b74d-af0f0c021571</span><br><span class="line">[root@container01 ~]# </span><br><span class="line"></span><br><span class="line">结果：使用kubectl logs -f spark-pi-driver 是可以看到的</span><br><span class="line"></span><br><span class="line">Pi is roughly 3.1451757258786293</span><br><span class="line">20/02/11 08:23:35 INFO SparkUI: Stopped Spark web UI at http://spark-pi-1581409376163-driver-svc.default.svc:4040</span><br><span class="line">20/02/11 08:23:35 INFO KubernetesClusterSchedulerBacke</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">1.上面的spark命令在 spark官网的基础上添加了 很多参数 官网上也能找到</span><br><span class="line">2.机器上要 先部署 jdk</span><br><span class="line"></span><br><span class="line">如果没有加 ：</span><br><span class="line">spark 出错了 使用 </span><br><span class="line"> 1.kubectl get all 先看看有没有error</span><br><span class="line"> 2.再查看spark （pod name） 通过 </span><br><span class="line"> kubectl logs -f spark-pi-driver </span><br><span class="line"> 3.如果出问题了 先 delete</span><br><span class="line"> kubectl delete [pod name] </span><br><span class="line"> [root@container01 ~]# kubectl delete pod/spark-pi-driver </span><br><span class="line"> pod "spark-pi-driver" deleted</span><br><span class="line"> [root@container01 ~]# </span><br><span class="line"> 4.再排错 重新提交</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200211162525671.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl8zOTU1NzUxMg==,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
</div>
</div>
</article>
<article id="post-Kudu-Impala故障案例01-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/01/05/Kudu-Impala%E6%95%85%E9%9A%9C%E6%A1%88%E4%BE%8B01-double-happy/">Kudu+Impala故障案例01--double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/01/05/Kudu-Impala%E6%95%85%E9%9A%9C%E6%A1%88%E4%BE%8B01-double-happy/" class="article-date">
<time datetime="2020-01-05T15:47:59.000Z" itemprop="datePublished">2020-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<p><strong>背景</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">Flink消费数据 最终写入Kudu里面进行存储，</span><br><span class="line">impala通过创建外部表的方式，关联kudu里的数据 进行后续的指标计算</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 1.Flink ->Sink kudu </span><br><span class="line"> 2.Impala ->关联kudu</span><br></pre></td></tr></table></figure></div>
<p><strong>故障</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">1.Flink sink kudu 表的名字 </span><br><span class="line"> eg： kudu01.test01 这个表 </span><br><span class="line"> Impala关联 kudu 之后 impala里的表是 kudu_test01</span><br><span class="line"></span><br><span class="line">即：</span><br><span class="line"> kudu01.test01 --> kudu_test01</span><br><span class="line"></span><br><span class="line">故障：</span><br><span class="line"> 当把kudu01.test01 表删掉 (通过kudu api 删掉表) </span><br><span class="line"> 重新创建的 kudu01.test01 </span><br><span class="line">删掉kudu表直接重建 但是 impala没有重新关联 会造成 出现 null</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">[vm24:21000] > select * from kudu_access08 limit 100;</span><br><span class="line">Query: select * from kudu_access08 limit 100</span><br><span class="line">Query submitted at: 2019-12-31 11:31:55 (Coordinator: http://vm24:25000)</span><br><span class="line">Query progress can be monitored at: http://vm24:25000/query_plan?query_id=f24990121ccc11cb:91d30b3d00000000</span><br><span class="line">+-------------+---------------+---------+---------------+</span><br><span class="line">| id | domain | traffic | time |</span><br><span class="line">+-------------+---------------+---------+---------------+</span><br><span class="line">| 0_0_3_4_4_9 | NULL | NULL | NULL |</span><br><span class="line">| 0_0_3_9_7_7 | NULL | NULL | 1577763046142 |</span><br><span class="line">| 0_0_5_1_2_3 | zhibo8.cc | 632 | NULL |</span><br><span class="line">| 0_0_8_3_3_0 | zhibo8.cc | 349 | 1577763045141 |</span><br><span class="line">| 0_0_8_8_0_2 | NULL | NULL | 1577763046142 |</span><br><span class="line">| 0_1_5_8_6_8 | NULL | 256 | NULL |</span><br><span class="line">| 0_1_9_0_6_1 | ruozedata.com | 513 | 1577763045141 |</span><br><span class="line">| 0_1_9_3_7_8 | dongqiudi.com | NULL | NULL |</span><br><span class="line">| 0_2_0_0_2_1 | NULL | 1212 | NULL |</span><br><span class="line">| 0_2_0_7_7_8 | NULL | 1323 | 1577763045141 |</span><br><span class="line">| 0_2_3_1_3_9 | ruozedata.com | NULL | NULL |</span><br><span class="line">| 0_2_6_2_9_3 | ruozedata.com | NULL | 1577763043139 |</span><br><span class="line">| 0_2_6_8_8_9 | NULL | 956 | 1577763046142 |</span><br><span class="line">| 0_2_7_2_2_7 | NULL | NULL | NULL |</span><br><span class="line">| 0_2_7_7_4_2 | zhibo8.cc | NULL | 1577763046142 |</span><br><span class="line">| 0_3_2_3_2_2 | dongqiudi.com | 309 | NULL |</span><br><span class="line">| 0_3_2_6_5_6 | NULL | NULL | NULL |</span><br><span class="line">| 0_3_8_9_8_0 | NULL | NULL | 1577763043139 |</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">删掉kudu表直接重建 但是 impala没有重新关联 会造成 出现 null</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">目前解决办法就是 ：</span><br><span class="line"> 通过impala把 impala关联的kudu表删掉 + 再把kudu里的表删掉 两个都重新创建 </span><br><span class="line"></span><br><span class="line">因为kudu+impala 刚刚接触 慢慢摸索当中</span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-Flink04-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/01/05/Flink04-double-happy/">Flink04--double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/01/05/Flink04-double-happy/" class="article-date">
<time datetime="2020-01-05T15:47:43.000Z" itemprop="datePublished">2020-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line">面试题：谈谈你对Flink ON YARN执行流程的理解</span><br><span class="line"></span><br><span class="line">acks=all 数据一定不会丢失吗</span><br><span class="line">如何保证Kafka消费者数据全局有序？ 伪命题</span><br><span class="line"> producer</span><br><span class="line"> storage</span><br><span class="line"> consumer</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">10分区 3个消费者</span><br><span class="line"> 哪个分区被谁消费</span><br><span class="line"></span><br><span class="line">C0</span><br><span class="line">C1</span><br><span class="line"></span><br><span class="line">T0</span><br><span class="line"> 0 1 2</span><br><span class="line">T1</span><br><span class="line"> 0 1 2</span><br><span class="line"></span><br><span class="line">6/2</span><br><span class="line">We then divide the number of partitions by the total number of</span><br><span class="line">consumers to determine the number of partitions to assign to each consumer. If it does not evenly</span><br><span class="line">divide, then the first few consumers will have one extra partition.</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">t0p0, t1p0, t1p1, t2p0, t2p1, t2p2</span><br><span class="line"></span><br><span class="line">C0 is subscribed to t0; t0p0</span><br><span class="line">C1 is subscribed to t0, t1; t0p0, t1p0, t1p1</span><br><span class="line">C2 is subscribed to t0, t1, t2. t0p0, t1p0, t1p1, t2p0, t2p1, t2p2</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">Sinl 到 HBase：</span><br><span class="line"> 1种.自定义 RichSinkFunction 或者 带并行的</span><br><span class="line"> 2种 . Hadoop Compatibility Beta</span><br><span class="line"> You can:</span><br><span class="line"> use Hadoop’s Writable data types in Flink programs.</span><br><span class="line"> use any Hadoop InputFormat as a DataSource.</span><br><span class="line"> use any Hadoop OutputFormat as a DataSink.</span><br><span class="line"> use a Hadoop Mapper as FlatMapFunction.</span><br><span class="line"> use a Hadoop Reducer as GroupReduceFunction.</span><br></pre></td></tr></table></figure></div>
<p><strong>Hadoop Compatibility Beta</strong><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/batch/hadoop_compatibility.html#hadoop-compatibility-beta" target="_blank" rel="noopener">Hadoop Compatibility Beta</a></p>
<p><img src="https://img-blog.csdnimg.cn/20200105201512668.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">也就是说 ：</span><br><span class="line"> 写入HBase 有两种方式</span><br><span class="line"> 1.自定义 Sink</span><br><span class="line"> 2.Flink-Hadoop 的Api</span><br></pre></td></tr></table></figure></div>
<p><strong>环境部署</strong><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/flinkDev/building.html#building-flink-from-source" target="_blank" rel="noopener">Building Flink from Source</a></p>
<p>部署：<br><img src="https://img-blog.csdnimg.cn/20200105202554890.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">部署：</span><br><span class="line"> 1.本地的 </span><br><span class="line"> 2.yarn</span><br><span class="line">作为重点。</span><br></pre></td></tr></table></figure></div>
<p><strong>Local</strong><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/getting-started/tutorials/local_setup.html#local-setup-tutorial" target="_blank" rel="noopener">Local Setup Tutorial</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">1.启动集群：</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$ bin/start-cluster.sh </span><br><span class="line">Starting cluster.</span><br><span class="line">Starting standalonesession daemon on host hadoop101.</span><br><span class="line">Starting taskexecutor daemon on host hadoop101.</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 1.Starting standalonesession </span><br><span class="line"> 2.Starting taskexecutor</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$ jps</span><br><span class="line">4688 TaskManagerRunner</span><br><span class="line">4722 Jps</span><br><span class="line">4258 StandaloneSessionClusterEntrypoint</span><br><span class="line">[double_happy@hadoop101 flink]$ </span><br><span class="line"></span><br><span class="line"></span><br><span class="line">2.看页面 端口 8081</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105203620172.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<p><img src="https://img-blog.csdnimg.cn/20200105203915909.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">按照官网给的案例：</span><br><span class="line"></span><br><span class="line">object SocketWindowWordCount {</span><br><span class="line"> def main(args: Array[String]) : Unit = {</span><br><span class="line"> val port: Int = try {</span><br><span class="line"> ParameterTool.fromArgs(args).getInt("port")</span><br><span class="line"> } catch {</span><br><span class="line"> case e: Exception => {</span><br><span class="line"> System.err.println("No port specified. Please run 'SocketWindowWordCount --host <host> --port <port>'")</span><br><span class="line"> return</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> val host: String = try {</span><br><span class="line"> ParameterTool.fromArgs(args).get("host")</span><br><span class="line"> } catch {</span><br><span class="line"> case e: Exception => {</span><br><span class="line"> System.err.println("No port specified. Please run 'SocketWindowWordCount --host <host> --port <port>'")</span><br><span class="line"> return</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"> </span><br><span class="line"></span><br><span class="line"> val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> val text = env.socketTextStream(host, port)</span><br><span class="line"> val windowCounts = text</span><br><span class="line"> .flatMap { w => w.split(",") }</span><br><span class="line"> .map { w => WordWithCount(w, 1) }</span><br><span class="line"> .keyBy("word")</span><br><span class="line"> .sum("count")</span><br><span class="line"></span><br><span class="line"> windowCounts.print().setParallelism(1)</span><br><span class="line"></span><br><span class="line"> env.execute("Socket Window WordCount")</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> case class WordWithCount(word: String, count: Long)</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 打包上传到本地</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1.启动 端口 nc ：nc -lk 9998</span><br><span class="line"></span><br><span class="line">2.提交flink jar 包</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br><span class="line">197</span><br><span class="line">198</span><br><span class="line">199</span><br><span class="line">200</span><br><span class="line">201</span><br><span class="line">202</span><br><span class="line">203</span><br><span class="line">204</span><br><span class="line">205</span><br><span class="line">206</span><br><span class="line">207</span><br><span class="line">208</span><br><span class="line">209</span><br><span class="line">210</span><br><span class="line">211</span><br><span class="line">212</span><br><span class="line">213</span><br><span class="line">214</span><br><span class="line">215</span><br><span class="line">216</span><br><span class="line">217</span><br><span class="line">218</span><br><span class="line">219</span><br><span class="line">220</span><br><span class="line">221</span><br><span class="line">222</span><br><span class="line">223</span><br><span class="line">224</span><br><span class="line">225</span><br><span class="line">226</span><br><span class="line">227</span><br><span class="line">228</span><br><span class="line">229</span><br><span class="line">230</span><br><span class="line">231</span><br><span class="line">232</span><br><span class="line">233</span><br><span class="line">234</span><br><span class="line">235</span><br><span class="line">236</span><br><span class="line">237</span><br><span class="line">238</span><br><span class="line">239</span><br><span class="line">240</span><br><span class="line">241</span><br><span class="line">242</span><br><span class="line">243</span><br><span class="line">244</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 bin]$ ./flink</span><br><span class="line">2020-01-05 23:20:00,751 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 23:20:00,751 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">./flink <ACTION> [OPTIONS] [ARGUMENTS]</span><br><span class="line"></span><br><span class="line">The following actions are available:</span><br><span class="line"></span><br><span class="line">Action "run" compiles and runs a program.</span><br><span class="line"></span><br><span class="line"> Syntax: run [OPTIONS] <jar-file> <arguments></span><br><span class="line"> "run" action options:</span><br><span class="line"> -c,--class <classname> Class with the program entry point</span><br><span class="line"> ("main()" method or "getPlan()"</span><br><span class="line"> method). Only needed if the JAR file</span><br><span class="line"> does not specify the class in its</span><br><span class="line"> manifest.</span><br><span class="line"> -C,--classpath <url> Adds a URL to each user code</span><br><span class="line"> classloader on all nodes in the</span><br><span class="line"> cluster. The paths must specify a</span><br><span class="line"> protocol (e.g. file://) and be</span><br><span class="line"> accessible on all nodes (e.g. by means</span><br><span class="line"> of a NFS share). You can use this</span><br><span class="line"> option multiple times for specifying</span><br><span class="line"> more than one URL. The protocol must</span><br><span class="line"> be supported by the {@link</span><br><span class="line"> java.net.URLClassLoader}.</span><br><span class="line"> -d,--detached If present, runs the job in detached</span><br><span class="line"> mode</span><br><span class="line"> -n,--allowNonRestoredState Allow to skip savepoint state that</span><br><span class="line"> cannot be restored. You need to allow</span><br><span class="line"> this if you removed an operator from</span><br><span class="line"> your program that was part of the</span><br><span class="line"> program when the savepoint was</span><br><span class="line"> triggered.</span><br><span class="line"> -p,--parallelism <parallelism> The parallelism with which to run the</span><br><span class="line"> program. Optional flag to override the</span><br><span class="line"> default value specified in the</span><br><span class="line"> configuration.</span><br><span class="line"> -py,--python <python> Python script with the program entry</span><br><span class="line"> point. The dependent resources can be</span><br><span class="line"> configured with the `--pyFiles`</span><br><span class="line"> option.</span><br><span class="line"> -pyfs,--pyFiles <pyFiles> Attach custom python files for job.</span><br><span class="line"> Comma can be used as the separator to</span><br><span class="line"> specify multiple files. The standard</span><br><span class="line"> python resource file suffixes such as</span><br><span class="line"> .py/.egg/.zip are all supported.(eg:</span><br><span class="line"> --pyFiles</span><br><span class="line"> file:///tmp/myresource.zip,hdfs:///$na</span><br><span class="line"> menode_address/myresource2.zip)</span><br><span class="line"> -pym,--pyModule <pyModule> Python module with the program entry</span><br><span class="line"> point. This option must be used in</span><br><span class="line"> conjunction with `--pyFiles`.</span><br><span class="line"> -q,--sysoutLogging If present, suppress logging output to</span><br><span class="line"> standard out.</span><br><span class="line"> -s,--fromSavepoint <savepointPath> Path to a savepoint to restore the job</span><br><span class="line"> from (for example</span><br><span class="line"> hdfs:///flink/savepoint-1537).</span><br><span class="line"> -sae,--shutdownOnAttachedExit If the job is submitted in attached</span><br><span class="line"> mode, perform a best-effort cluster</span><br><span class="line"> shutdown when the CLI is terminated</span><br><span class="line"> abruptly, e.g., in response to a user</span><br><span class="line"> interrupt, such as typing Ctrl + C.</span><br><span class="line"> Options for yarn-cluster mode:</span><br><span class="line"> -d,--detached If present, runs the job in detached</span><br><span class="line"> mode</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to</span><br><span class="line"> which to connect. Use this flag to</span><br><span class="line"> connect to a different JobManager than</span><br><span class="line"> the one specified in the</span><br><span class="line"> configuration.</span><br><span class="line"> -sae,--shutdownOnAttachedExit If the job is submitted in attached</span><br><span class="line"> mode, perform a best-effort cluster</span><br><span class="line"> shutdown when the CLI is terminated</span><br><span class="line"> abruptly, e.g., in response to a user</span><br><span class="line"> interrupt, such as typing Ctrl + C.</span><br><span class="line"> -yat,--yarnapplicationType <arg> Set a custom application type for the</span><br><span class="line"> application on YARN</span><br><span class="line"> -yD <property=value> use value for given property</span><br><span class="line"> -yd,--yarndetached If present, runs the job in detached</span><br><span class="line"> mode (deprecated; use non-YARN</span><br><span class="line"> specific option instead)</span><br><span class="line"> -yh,--yarnhelp Help for the Yarn session CLI.</span><br><span class="line"> -yid,--yarnapplicationId <arg> Attach to running YARN session</span><br><span class="line"> -yj,--yarnjar <arg> Path to Flink jar file</span><br><span class="line"> -yjm,--yarnjobManagerMemory <arg> Memory for JobManager Container with</span><br><span class="line"> optional unit (default: MB)</span><br><span class="line"> -yn,--yarncontainer <arg> Number of YARN container to allocate</span><br><span class="line"> (=Number of Task Managers)</span><br><span class="line"> -ynl,--yarnnodeLabel <arg> Specify YARN node label for the YARN</span><br><span class="line"> application</span><br><span class="line"> -ynm,--yarnname <arg> Set a custom name for the application</span><br><span class="line"> on YARN</span><br><span class="line"> -yq,--yarnquery Display available YARN resources</span><br><span class="line"> (memory, cores)</span><br><span class="line"> -yqu,--yarnqueue <arg> Specify YARN queue.</span><br><span class="line"> -ys,--yarnslots <arg> Number of slots per TaskManager</span><br><span class="line"> -yst,--yarnstreaming Start Flink in streaming mode</span><br><span class="line"> -yt,--yarnship <arg> Ship files in the specified directory</span><br><span class="line"> (t for transfer)</span><br><span class="line"> -ytm,--yarntaskManagerMemory <arg> Memory per TaskManager Container with</span><br><span class="line"> optional unit (default: MB)</span><br><span class="line"> -yz,--yarnzookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"></span><br><span class="line"> Options for default mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which</span><br><span class="line"> to connect. Use this flag to connect to a</span><br><span class="line"> different JobManager than the one specified</span><br><span class="line"> in the configuration.</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths</span><br><span class="line"> for high availability mode</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Action "info" shows the optimized execution plan of the program (JSON).</span><br><span class="line"></span><br><span class="line"> Syntax: info [OPTIONS] <jar-file> <arguments></span><br><span class="line"> "info" action options:</span><br><span class="line"> -c,--class <classname> Class with the program entry point</span><br><span class="line"> ("main()" method or "getPlan()" method).</span><br><span class="line"> Only needed if the JAR file does not</span><br><span class="line"> specify the class in its manifest.</span><br><span class="line"> -p,--parallelism <parallelism> The parallelism with which to run the</span><br><span class="line"> program. Optional flag to override the</span><br><span class="line"> default value specified in the</span><br><span class="line"> configuration.</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Action "list" lists running and scheduled programs.</span><br><span class="line"></span><br><span class="line"> Syntax: list [OPTIONS]</span><br><span class="line"> "list" action options:</span><br><span class="line"> -r,--running Show only running programs and their JobIDs</span><br><span class="line"> -s,--scheduled Show only scheduled programs and their JobIDs</span><br><span class="line"> Options for yarn-cluster mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to</span><br><span class="line"> which to connect. Use this flag to connect</span><br><span class="line"> to a different JobManager than the one</span><br><span class="line"> specified in the configuration.</span><br><span class="line"> -yid,--yarnapplicationId <arg> Attach to running YARN session</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"></span><br><span class="line"> Options for default mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which</span><br><span class="line"> to connect. Use this flag to connect to a</span><br><span class="line"> different JobManager than the one specified</span><br><span class="line"> in the configuration.</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths</span><br><span class="line"> for high availability mode</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Action "stop" stops a running program with a savepoint (streaming jobs only).</span><br><span class="line"></span><br><span class="line"> Syntax: stop [OPTIONS] <Job ID></span><br><span class="line"> "stop" action options:</span><br><span class="line"> -d,--drain Send MAX_WATERMARK before taking the</span><br><span class="line"> savepoint and stopping the pipelne.</span><br><span class="line"> -p,--savepointPath <savepointPath> Path to the savepoint (for example</span><br><span class="line"> hdfs:///flink/savepoint-1537). If no</span><br><span class="line"> directory is specified, the configured</span><br><span class="line"> default will be used</span><br><span class="line"> ("state.savepoints.dir").</span><br><span class="line"> Options for yarn-cluster mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to</span><br><span class="line"> which to connect. Use this flag to connect</span><br><span class="line"> to a different JobManager than the one</span><br><span class="line"> specified in the configuration.</span><br><span class="line"> -yid,--yarnapplicationId <arg> Attach to running YARN session</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"></span><br><span class="line"> Options for default mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which</span><br><span class="line"> to connect. Use this flag to connect to a</span><br><span class="line"> different JobManager than the one specified</span><br><span class="line"> in the configuration.</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths</span><br><span class="line"> for high availability mode</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Action "cancel" cancels a running program.</span><br><span class="line"></span><br><span class="line"> Syntax: cancel [OPTIONS] <Job ID></span><br><span class="line"> "cancel" action options:</span><br><span class="line"> -s,--withSavepoint <targetDirectory> **DEPRECATION WARNING**: Cancelling</span><br><span class="line"> a job with savepoint is deprecated.</span><br><span class="line"> Use "stop" instead.</span><br><span class="line"> Trigger savepoint and cancel job.</span><br><span class="line"> The target directory is optional. If</span><br><span class="line"> no directory is specified, the</span><br><span class="line"> configured default directory</span><br><span class="line"> (state.savepoints.dir) is used.</span><br><span class="line"> Options for yarn-cluster mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to</span><br><span class="line"> which to connect. Use this flag to connect</span><br><span class="line"> to a different JobManager than the one</span><br><span class="line"> specified in the configuration.</span><br><span class="line"> -yid,--yarnapplicationId <arg> Attach to running YARN session</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"></span><br><span class="line"> Options for default mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which</span><br><span class="line"> to connect. Use this flag to connect to a</span><br><span class="line"> different JobManager than the one specified</span><br><span class="line"> in the configuration.</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths</span><br><span class="line"> for high availability mode</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Action "savepoint" triggers savepoints for a running job or disposes existing ones.</span><br><span class="line"></span><br><span class="line"> Syntax: savepoint [OPTIONS] <Job ID> [<target directory>]</span><br><span class="line"> "savepoint" action options:</span><br><span class="line"> -d,--dispose <arg> Path of savepoint to dispose.</span><br><span class="line"> -j,--jarfile <jarfile> Flink program JAR file.</span><br><span class="line"> Options for yarn-cluster mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to</span><br><span class="line"> which to connect. Use this flag to connect</span><br><span class="line"> to a different JobManager than the one</span><br><span class="line"> specified in the configuration.</span><br><span class="line"> -yid,--yarnapplicationId <arg> Attach to running YARN session</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper</span><br><span class="line"> sub-paths for high availability mode</span><br><span class="line"></span><br><span class="line"> Options for default mode:</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which</span><br><span class="line"> to connect. Use this flag to connect to a</span><br><span class="line"> different JobManager than the one specified</span><br><span class="line"> in the configuration.</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths</span><br><span class="line"> for high availability mode</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Please specify an action.</span><br><span class="line">[double_happy@hadoop101 bin]$</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">./flink run [OPTIONS] <jar-file> <arguments> </span><br><span class="line"></span><br><span class="line">注意： 提交jar 运行 命令</span><br><span class="line"></span><br><span class="line">./flink run \</span><br><span class="line">--class com.sx.flink04.SocketWindowWordCount \</span><br><span class="line">/home/double_happy/lib/Flink-1.0.jar \</span><br><span class="line">--port 9998 \</span><br><span class="line">--host hadoop101</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">提交结果：</span><br><span class="line">[double_happy@hadoop101 bin]$ ./flink run \</span><br><span class="line">> --class com.sx.flink04.SocketWindowWordCount \</span><br><span class="line">> /home/double_happy/lib/Flink-1.0.jar \</span><br><span class="line">> --port 9998 \</span><br><span class="line">> --host hadoop101</span><br><span class="line">Starting execution of program</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105210441990.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><strong>点进去</strong><br><img src="https://img-blog.csdnimg.cn/20200105210608282.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">nc 生产一些数据：</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 ~]$ nc -lk 9998</span><br><span class="line">a,a,a,a,bb,c</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">那么这些数据在Flink 哪里呢？</span><br><span class="line"></span><br><span class="line">点击 Task Managers： 能看到很多参数的 ***</span><br><span class="line"> 1.能看到 jvm的 参数 </span><br><span class="line"> 2.Network 相关的</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105210959821.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">这些日志哪里来的？</span><br><span class="line"> 就是从文件系统读进来的</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105211034704.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">结果：</span><br><span class="line"> 1.通过 页面可以看的到的 （就是从本地结果 传到页面上的 ）</span><br><span class="line"> 2.通过本地也能看到</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">本地查看：</span><br><span class="line"> [double_happy@hadoop101 log]$ tail -200f flink-double_happy-taskexecutor-1-hadoop101.out</span><br><span class="line"> WordWithCount(a,1)</span><br><span class="line"> WordWithCount(a,2)</span><br><span class="line"> WordWithCount(a,3)</span><br><span class="line"> WordWithCount(a,4)</span><br><span class="line"> WordWithCount(bb,1)</span><br><span class="line"> WordWithCount(c,1)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> $ tail -f log/flink-*-taskexecutor-*.out </span><br><span class="line">就可以看到本地 结果</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">如何停止这个作业：</span><br><span class="line"> 1.页面 上 running jos 点进去 </span><br><span class="line"> 2.本地命令</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105211557264.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">本地命令：</span><br><span class="line"> ./flink cancel [OPTIONS] <Job ID></span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ ./flink cancel 710a899ba9589d6a688c14931f7ef6cd</span><br><span class="line">Cancelling job 710a899ba9589d6a688c14931f7ef6cd.</span><br><span class="line">Cancelled job 710a899ba9589d6a688c14931f7ef6cd.</span><br><span class="line">[double_happy@hadoop101 bin]$</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105212016595.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">关闭集群：</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ jps</span><br><span class="line">4688 TaskManagerRunner</span><br><span class="line">4258 StandaloneSessionClusterEntrypoint</span><br><span class="line">9444 Jps</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ ./stop-cluster.sh </span><br><span class="line">Stopping taskexecutor daemon (pid: 4688) on host hadoop101.</span><br><span class="line">Stopping standalonesession daemon (pid: 4258) on host hadoop101.</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ jps</span><br><span class="line">10457 Jps</span><br><span class="line">[double_happy@hadoop101 bin]$</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 页面也就打不开了</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">以上：</span><br><span class="line"> 就是最简单的local模式 运行jar job</span><br></pre></td></tr></table></figure></div>
<h2 id="对比Spark-shell"><a href="#对比Spark-shell" class="headerlink" title="对比Spark-shell"></a>对比Spark-shell</h2><p><strong>start-scala-shell.sh</strong> </p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 bin]$ ./start-scala-shell.sh </span><br><span class="line">Starting Flink Shell:</span><br><span class="line">log4j:WARN No appenders could be found for logger (org.apache.flink.configuration.GlobalConfiguration).</span><br><span class="line">log4j:WARN Please initialize the log4j system properly.</span><br><span class="line">log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.</span><br><span class="line">Error: please specify execution mode:</span><br><span class="line">[local | remote <host> <port> | yarn]</span><br><span class="line">[double_happy@hadoop101 bin]$ </span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> Error: please specify execution mode: 指定跑在什么模式下 </span><br><span class="line"> Spark和Flink都可以跑在多个模式上的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 bin]$ ./start-scala-shell.sh local</span><br><span class="line">Starting Flink Shell:</span><br><span class="line"></span><br><span class="line">Starting local Flink cluster (host: localhost, port: 8081).</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Connecting to Flink cluster (host: localhost, port: 8081).</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> ▒▓██▓██▒</span><br><span class="line"> ▓████▒▒█▓▒▓███▓▒</span><br><span class="line"> ▓███▓░░ ▒▒▒▓██▒ ▒</span><br><span class="line"> ░██▒ ▒▒▓▓█▓▓▒░ ▒████</span><br><span class="line"> ██▒ ░▒▓███▒ ▒█▒█▒</span><br><span class="line"> ░▓█ ███ ▓░▒██</span><br><span class="line"> ▓█ ▒▒▒▒▒▓██▓░▒░▓▓█</span><br><span class="line"> █░ █ ▒▒░ ███▓▓█ ▒█▒▒▒</span><br><span class="line"> ████░ ▒▓█▓ ██▒▒▒ ▓███▒</span><br><span class="line"> ░▒█▓▓██ ▓█▒ ▓█▒▓██▓ ░█░</span><br><span class="line"> ▓░▒▓████▒ ██ ▒█ █▓░▒█▒░▒█▒</span><br><span class="line"> ███▓░██▓ ▓█ █ █▓ ▒▓█▓▓█▒</span><br><span class="line"> ░██▓ ░█░ █ █▒ ▒█████▓▒ ██▓░▒</span><br><span class="line"> ███░ ░ █░ ▓ ░█ █████▒░░ ░█░▓ ▓░</span><br><span class="line"> ██▓█ ▒▒▓▒ ▓███████▓░ ▒█▒ ▒▓ ▓██▓</span><br><span class="line"> ▒██▓ ▓█ █▓█ ░▒█████▓▓▒░ ██▒▒ █ ▒ ▓█▒</span><br><span class="line"> ▓█▓ ▓█ ██▓ ░▓▓▓▓▓▓▓▒ ▒██▓ ░█▒</span><br><span class="line"> ▓█ █ ▓███▓▒░ ░▓▓▓███▓ ░▒░ ▓█</span><br><span class="line"> ██▓ ██▒ ░▒▓▓███▓▓▓▓▓██████▓▒ ▓███ █</span><br><span class="line">▓███▒ ███ ░▓▓▒░░ ░▓████▓░ ░▒▓▒ █▓</span><br><span class="line">█▓▒▒▓▓██ ░▒▒░░░▒▒▒▒▓██▓░ █▓</span><br><span class="line">██ ▓░▒█ ▓▓▓▓▒░░ ▒█▓ ▒▓▓██▓ ▓▒ ▒▒▓</span><br><span class="line">▓█▓ ▓▒█ █▓░ ░▒▓▓██▒ ░▓█▒ ▒▒▒░▒▒▓█████▒</span><br><span class="line"> ██░ ▓█▒█▒ ▒▓▓▒ ▓█ █░ ░░░░ ░█▒</span><br><span class="line"> ▓█ ▒█▓ ░ █░ ▒█ █▓</span><br><span class="line"> █▓ ██ █░ ▓▓ ▒█▓▓▓▒█░</span><br><span class="line"> █▓ ░▓██░ ▓▒ ▓█▓▒░░░▒▓█░ ▒█</span><br><span class="line"> ██ ▓█▓░ ▒ ░▒█▒██▒ ▓▓</span><br><span class="line"> ▓█▒ ▒█▓▒░ ▒▒ █▒█▓▒▒░░▒██</span><br><span class="line"> ░██▒ ▒▓▓▒ ▓██▓▒█▒ ░▓▓▓▓▒█▓</span><br><span class="line"> ░▓██▒ ▓░ ▒█▓█ ░░▒▒▒</span><br><span class="line"> ▒▓▓▓▓▓▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒░░▓▓ ▓░▒█░</span><br><span class="line"></span><br><span class="line"> F L I N K - S C A L A - S H E L L</span><br><span class="line"></span><br><span class="line">NOTE: Use the prebound Execution Environments and Table Environment to implement batch or streaming programs.</span><br><span class="line"></span><br><span class="line"> Batch - Use the 'benv' and 'btenv' variable</span><br><span class="line"></span><br><span class="line"> * val dataSet = benv.readTextFile("/path/to/data")</span><br><span class="line"> * dataSet.writeAsText("/path/to/output")</span><br><span class="line"> * benv.execute("My batch program")</span><br><span class="line"> *</span><br><span class="line"> * val batchTable = btenv.fromDataSet(dataSet)</span><br><span class="line"> * btenv.registerTable("tableName", batchTable)</span><br><span class="line"> * val result = btenv.sqlQuery("SELECT * FROM tableName").collect</span><br><span class="line"> HINT: You can use print() on a DataSet to print the contents or collect()</span><br><span class="line"> a sql query result back to the shell.</span><br><span class="line"></span><br><span class="line"> Streaming - Use the 'senv' and 'stenv' variable</span><br><span class="line"></span><br><span class="line"> * val dataStream = senv.fromElements(1, 2, 3, 4)</span><br><span class="line"> * dataStream.countWindowAll(2).sum(0).print()</span><br><span class="line"> *</span><br><span class="line"> * val streamTable = stenv.fromDataStream(dataStream, 'num)</span><br><span class="line"> * val resultTable = streamTable.select('num).where('num % 2 === 1 )</span><br><span class="line"> * resultTable.toAppendStream[Row].print()</span><br><span class="line"> * senv.execute("My streaming program")</span><br><span class="line"> HINT: You can only print a DataStream to the shell in local mode.</span><br><span class="line"> </span><br><span class="line"></span><br><span class="line">scala></span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> Batch - Use the 'benv' and 'btenv' variable</span><br><span class="line"> Streaming - Use the 'senv' and 'stenv' variable</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">批处理：</span><br><span class="line">scala> val text = benv.fromElements("kairis,kairis,kairis","double_happy,sxwang")</span><br><span class="line">text: org.apache.flink.api.scala.DataSet[String] = org.apache.flink.api.scala.DataSet@26a004ed</span><br><span class="line"></span><br><span class="line">scala> text.flatMap(_.split(",")).map((_,1)).groupBy(0).sum(1).print</span><br><span class="line">(double_happy,1)</span><br><span class="line">(kairis,3)</span><br><span class="line">(sxwang,1)</span><br><span class="line"></span><br><span class="line">scala></span><br></pre></td></tr></table></figure></div>
<h2 id="跑在Yarn上"><a href="#跑在Yarn上" class="headerlink" title="跑在Yarn上 ***"></a>跑在Yarn上 ***</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">面试题：谈谈你对Flink ON YARN执行流程的理解？</span><br><span class="line"></span><br><span class="line"> 和Spark mr 都差不多 只是 ApplicaitonMaster 名字变了</span><br></pre></td></tr></table></figure></div>
<p><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/deployment/yarn_setup.html#yarn-setup" target="_blank" rel="noopener">YARN Setup</a><br><img src="https://img-blog.csdnimg.cn/20200105214018381.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/deployment/yarn_setup.html#background--internals" target="_blank" rel="noopener">Background / Internals</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">The YARN client needs to access the Hadoop configuration to connect to the YARN resource manager and HDFS</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> YARN client 要配置gateway</span><br><span class="line"></span><br><span class="line">但是：</span><br><span class="line"> Flink跑yarn 跟Spark 跑yarn 有点区别</span><br><span class="line"></span><br><span class="line">为什么呢？</span><br><span class="line"> Flink有两种模式跑yarn</span><br><span class="line"> 1.Start a long-running Flink cluster on YARN</span><br><span class="line"> 启动一个长服务</span><br><span class="line"> 2.Run a Flink job on YARN</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105215637367.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">Spark里面 就是 右边这种 </span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> Spark申请资源是耗时的 还记得吗 依赖的 jar包需要传到hdfs上 也不然每次都要传 还记得么</span><br><span class="line"></span><br><span class="line">这两种方式会带来什么好处：</span><br><span class="line"> 1.长服务 </span><br><span class="line"> 省资源申请的时间</span><br></pre></td></tr></table></figure></div>
<p><strong>两种方式分别演示</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br></pre></td><td class="code"><pre><span class="line"> ./yarn-session.sh 运行的时候有坑的 ：</span><br><span class="line"> 1.你自己源码编译好的是没有问题的</span><br><span class="line"> 2.不是编译的 要在flink lib包下面 加一个 lib哦</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ ./yarn-session.sh --help</span><br><span class="line">2020-01-05 22:32:14,068 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.rpc.address, hadoop101</span><br><span class="line">2020-01-05 22:32:14,070 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.rpc.port, 6123</span><br><span class="line">2020-01-05 22:32:14,070 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.heap.size, 1024m</span><br><span class="line">2020-01-05 22:32:14,070 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: taskmanager.heap.size, 1024m</span><br><span class="line">2020-01-05 22:32:14,070 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: taskmanager.numberOfTaskSlots, 1</span><br><span class="line">2020-01-05 22:32:14,070 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: parallelism.default, 1</span><br><span class="line">2020-01-05 22:32:14,071 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.execution.failover-strategy, region</span><br><span class="line">2020-01-05 22:32:14,071 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: rest.port, 8081</span><br><span class="line">2020-01-05 22:32:14,667 WARN org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">2020-01-05 22:32:14,923 INFO org.apache.flink.runtime.security.modules.HadoopModule - Hadoop user set to double_happy (auth:SIMPLE)</span><br><span class="line">Usage:</span><br><span class="line"> Required</span><br><span class="line"> -n,--container <arg> Number of YARN container to allocate (=Number of Task Managers)</span><br><span class="line"> Optional</span><br><span class="line"> -at,--applicationType <arg> Set a custom application type for the application on YARN</span><br><span class="line"> -D <property=value> use value for given property</span><br><span class="line"> -d,--detached If present, runs the job in detached mode</span><br><span class="line"> -h,--help Help for the Yarn session CLI.</span><br><span class="line"> -id,--applicationId <arg> Attach to running YARN session</span><br><span class="line"> -j,--jar <arg> Path to Flink jar file</span><br><span class="line"> -jm,--jobManagerMemory <arg> Memory for JobManager Container with optional unit (default: MB)</span><br><span class="line"> -m,--jobmanager <arg> Address of the JobManager (master) to which to connect. Use this flag to connect to a different JobManager than the one specified in the configuration.</span><br><span class="line"> -n,--container <arg> Number of YARN container to allocate (=Number of Task Managers)</span><br><span class="line"> -nl,--nodeLabel <arg> Specify YARN node label for the YARN application</span><br><span class="line"> -nm,--name <arg> Set a custom name for the application on YARN</span><br><span class="line"> -q,--query Display available YARN resources (memory, cores)</span><br><span class="line"> -qu,--queue <arg> Specify YARN queue.</span><br><span class="line"> -s,--slots <arg> Number of slots per TaskManager</span><br><span class="line"> -sae,--shutdownOnAttachedExit If the job is submitted in attached mode, perform a best-effort cluster shutdown when the CLI is terminated abruptly, e.g., in response to a user interrupt, such</span><br><span class="line"> as typing Ctrl + C.</span><br><span class="line"> -st,--streaming Start Flink in streaming mode</span><br><span class="line"> -t,--ship <arg> Ship files in the specified directory (t for transfer)</span><br><span class="line"> -tm,--taskManagerMemory <arg> Memory per TaskManager Container with optional unit (default: MB)</span><br><span class="line"> -yd,--yarndetached If present, runs the job in detached mode (deprecated; use non-YARN specific option instead)</span><br><span class="line"> -z,--zookeeperNamespace <arg> Namespace to create the Zookeeper sub-paths for high availability mode</span><br><span class="line">[double_happy@hadoop101 bin]$</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"> Required</span><br><span class="line"> -n,--container <arg> Number of YARN container to allocate (=Number of Task Managers)</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 必填的 </span><br><span class="line"> -n 就是 container = Number of Task Managers </span><br><span class="line"></span><br><span class="line">不是必填的：</span><br><span class="line"> -jm,--jobManagerMemory <arg> Memory for JobManager Container with optional unit (default: MB)</span><br><span class="line"> -tm,--taskManagerMemory <arg> Memory per TaskManager Container with optional unit (default: MB)</span><br><span class="line"> -d,--detached If present, runs the job in detached mode 运行你的job 以什么方式</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">运行起来：</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 bin]$ ./yarn-session.sh -n 2 -jm 1024 -tm 1024 -d </span><br><span class="line"></span><br><span class="line">2020-01-05 22:37:43,651 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.rpc.address, hadoop101</span><br><span class="line">2020-01-05 22:37:43,656 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.rpc.port, 6123</span><br><span class="line">2020-01-05 22:37:43,656 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.heap.size, 1024m</span><br><span class="line">2020-01-05 22:37:43,657 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: taskmanager.heap.size, 1024m</span><br><span class="line">2020-01-05 22:37:43,657 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: taskmanager.numberOfTaskSlots, 1</span><br><span class="line">2020-01-05 22:37:43,657 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: parallelism.default, 1</span><br><span class="line">2020-01-05 22:37:43,658 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: jobmanager.execution.failover-strategy, region</span><br><span class="line">2020-01-05 22:37:43,658 INFO org.apache.flink.configuration.GlobalConfiguration - Loading configuration property: rest.port, 8081</span><br><span class="line">2020-01-05 22:37:44,328 WARN org.apache.hadoop.util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable</span><br><span class="line">2020-01-05 22:37:44,505 INFO org.apache.flink.runtime.security.modules.HadoopModule - Hadoop user set to double_happy (auth:SIMPLE)</span><br><span class="line">2020-01-05 22:37:44,611 INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at /0.0.0.0:8032</span><br><span class="line">2020-01-05 22:37:44,761 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - The argument n is deprecated in will be ignored.</span><br><span class="line">2020-01-05 22:37:44,930 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Cluster specification: ClusterSpecification{masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=2, slotsPerTaskManager=1}</span><br><span class="line">2020-01-05 22:37:45,573 WARN org.apache.flink.yarn.AbstractYarnClusterDescriptor - The configuration directory ('/home/double_happy/app/flink-1.9.1/conf') contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">2020-01-05 22:37:47,853 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Submitting application master application_1578233120766_0002</span><br><span class="line">2020-01-05 22:37:48,255 INFO org.apache.hadoop.yarn.client.api.impl.YarnClientImpl - Submitted application application_1578233120766_0002</span><br><span class="line">2020-01-05 22:37:48,255 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Waiting for the cluster to be allocated</span><br><span class="line">2020-01-05 22:37:48,257 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Deploying cluster, current state ACCEPTED</span><br><span class="line">2020-01-05 22:37:58,915 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - YARN application has been deployed successfully.</span><br><span class="line">2020-01-05 22:37:58,915 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - The Flink YARN client has been started in detached mode. In order to stop Flink on YARN, use the following command or a YARN web interface to stop it:</span><br><span class="line">yarn application -kill application_1578233120766_0002</span><br><span class="line">Please also note that the temporary files of the YARN session in the home directory will not be removed.</span><br><span class="line">2020-01-05 22:37:59,734 INFO org.apache.flink.runtime.rest.RestClient - Rest client endpoint started.</span><br><span class="line">Flink JobManager is now running on hadoop101:8081 with leader id 00000000-0000-0000-0000-000000000000.</span><br><span class="line">JobManager Web Interface: http://hadoop101:8081</span><br><span class="line">2020-01-05 22:37:59,757 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - The Flink YARN client has been started in detached mode. In order to stop Flink on YARN, use the following command or a YARN web interface to stop it:</span><br><span class="line">yarn application -kill application_1578233120766_0002</span><br><span class="line">[double_happy@hadoop101 bin]$</span><br></pre></td></tr></table></figure></div>
<p>去yarn 上查看：<br><img src="https://img-blog.csdnimg.cn/20200105223922233.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">运行一个 例子试试：</span><br><span class="line"> Example</span><br><span class="line"></span><br><span class="line">wget -O LICENSE-2.0.txt http://www.apache.org/licenses/LICENSE-2.0.txt</span><br><span class="line">hadoop fs -copyFromLocal LICENSE-2.0.txt hdfs:/// ...</span><br><span class="line">./bin/flink run ./examples/batch/WordCount.jar \</span><br><span class="line"> --input hdfs:///..../LICENSE-2.0.txt --output hdfs:///.../wordcount-result.txt</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 这是啥意思呀：</span><br><span class="line"> 就是把数据 上传到hdfs上 然后运行</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 data]$ hadoop fs -put ./LICENSE-2.0.txt /flink/input/</span><br><span class="line">put: `/flink/input/': No such file or directory</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 data]$ hadoop fs -mkdir -p /flink/input/</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 data]$ hadoop fs -put ./LICENSE-2.0.txt /flink/input/ </span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 data]$ hadoop fs -ls /flink/input/</span><br><span class="line">Found 1 items</span><br><span class="line">-rw-r--r-- 1 double_happy supergroup 11358 2020-01-05 22:43 /flink/input/LICENSE-2.0.txt</span><br><span class="line">[double_happy@hadoop101 data]$</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">运行：</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$ ./bin/flink run ./examples/batch/WordCount.jar \</span><br><span class="line">> --input hdfs://hadoop101:8020/flink/input/LICENSE-2.0.txt \</span><br><span class="line">> --output hdfs://hadoop101:8020/flink/output</span><br><span class="line">2020-01-05 22:46:46,517 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 22:46:46,517 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 22:46:47,119 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - YARN properties set default parallelism to 2</span><br><span class="line">2020-01-05 22:46:47,119 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - YARN properties set default parallelism to 2</span><br><span class="line">YARN properties set default parallelism to 2</span><br><span class="line">2020-01-05 22:46:47,238 INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at /0.0.0.0:8032</span><br><span class="line">2020-01-05 22:46:47,449 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 22:46:47,449 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 22:46:47,650 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Found application JobManager host name 'hadoop101' and port '8081' from supplied application id 'application_1578233120766_0002'</span><br><span class="line">Starting execution of program</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">结果：</span><br><span class="line">[double_happy@hadoop101 flink]$ hadoop fs -ls /flink/output</span><br><span class="line">-rw-r--r-- 1 double_happy supergroup 4499 2020-01-05 22:47 /flink/output</span><br><span class="line">[double_happy@hadoop101 flink]$ hadoop fs -text /flink/output</span><br><span class="line">0 3</span><br><span class="line">1 2</span><br><span class="line">2 4</span><br><span class="line">2004 1</span><br><span class="line">3 1</span><br><span class="line">4 1</span><br><span class="line">5 1</span><br><span class="line">50 1</span><br><span class="line">6 1</span><br><span class="line">7 1</span><br><span class="line">8 1</span><br><span class="line">9 2</span><br><span class="line">a 22</span><br><span class="line">above 1</span><br><span class="line">acceptance 1</span><br><span class="line">accepting 3</span><br><span class="line">act 1</span><br><span class="line">acting 1</span><br><span class="line">acts 1</span><br><span class="line">add 2</span><br><span class="line">addendum 1</span><br><span class="line">additional 5</span><br><span class="line">additions 1</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> Flink输出的 是文件 </span><br><span class="line"> Spark是文件夹</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> 这个模式 他怎么知道是提交到 哪个session上去呢？</span><br><span class="line"></span><br><span class="line">看控制台log：</span><br><span class="line">2020-01-05 22:46:46,517 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli </span><br><span class="line"> - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line"></span><br><span class="line">打开： /tmp/.yarn-properties-double_happy</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$ cat /tmp/.yarn-properties-double_happy</span><br><span class="line">#Generated YARN properties file</span><br><span class="line">#Sun Jan 05 22:37:59 CST 2020</span><br><span class="line">parallelism=2</span><br><span class="line">dynamicPropertiesString=</span><br><span class="line">applicationID=application_1578233120766_0002 //yarn 的 application的 id</span><br><span class="line">[double_happy@hadoop101 flink]$</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 这块 如果你已经使用了 这个模式跑 之后 还想使用 local跑 </span><br><span class="line"> 需要把/tmp/.yarn-properties-double_happy 这个 干掉 也不然 本地模式会 跑到yarn上 报错</span><br></pre></td></tr></table></figure></div>
<p><strong>Run a single Flink job on YARN</strong><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/deployment/yarn_setup.html#run-a-single-flink-job-on-yarn" target="_blank" rel="noopener">Run a single Flink job on YARN</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br><span class="line">113</span><br><span class="line">114</span><br><span class="line">115</span><br><span class="line">116</span><br><span class="line">117</span><br><span class="line">118</span><br><span class="line">119</span><br><span class="line">120</span><br><span class="line">121</span><br><span class="line">122</span><br><span class="line">123</span><br><span class="line">124</span><br><span class="line">125</span><br><span class="line">126</span><br><span class="line">127</span><br><span class="line">128</span><br><span class="line">129</span><br><span class="line">130</span><br><span class="line">131</span><br><span class="line">132</span><br><span class="line">133</span><br><span class="line">134</span><br><span class="line">135</span><br><span class="line">136</span><br><span class="line">137</span><br><span class="line">138</span><br><span class="line">139</span><br><span class="line">140</span><br><span class="line">141</span><br><span class="line">142</span><br><span class="line">143</span><br><span class="line">144</span><br><span class="line">145</span><br><span class="line">146</span><br><span class="line">147</span><br><span class="line">148</span><br><span class="line">149</span><br><span class="line">150</span><br><span class="line">151</span><br><span class="line">152</span><br><span class="line">153</span><br><span class="line">154</span><br><span class="line">155</span><br><span class="line">156</span><br><span class="line">157</span><br><span class="line">158</span><br><span class="line">159</span><br><span class="line">160</span><br><span class="line">161</span><br><span class="line">162</span><br><span class="line">163</span><br><span class="line">164</span><br><span class="line">165</span><br><span class="line">166</span><br><span class="line">167</span><br><span class="line">168</span><br><span class="line">169</span><br><span class="line">170</span><br><span class="line">171</span><br><span class="line">172</span><br><span class="line">173</span><br><span class="line">174</span><br><span class="line">175</span><br><span class="line">176</span><br><span class="line">177</span><br><span class="line">178</span><br><span class="line">179</span><br><span class="line">180</span><br><span class="line">181</span><br><span class="line">182</span><br><span class="line">183</span><br><span class="line">184</span><br><span class="line">185</span><br><span class="line">186</span><br><span class="line">187</span><br><span class="line">188</span><br><span class="line">189</span><br><span class="line">190</span><br><span class="line">191</span><br><span class="line">192</span><br><span class="line">193</span><br><span class="line">194</span><br><span class="line">195</span><br><span class="line">196</span><br></pre></td><td class="code"><pre><span class="line">官方案例：</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$ ./bin/flink run -m yarn-cluster ./examples/batch/WordCount.jar</span><br><span class="line">2020-01-05 23:06:06,559 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 23:06:06,559 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 23:06:07,151 INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at /0.0.0.0:8032</span><br><span class="line">2020-01-05 23:06:07,324 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 23:06:07,324 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 23:06:07,520 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Cluster specification: ClusterSpecification{masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=1, slotsPerTaskManager=1}</span><br><span class="line">2020-01-05 23:06:08,127 WARN org.apache.flink.yarn.AbstractYarnClusterDescriptor - The configuration directory ('/home/double_happy/app/flink-1.9.1/conf') contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">2020-01-05 23:06:10,927 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Submitting application master application_1578233120766_0003</span><br><span class="line">2020-01-05 23:06:10,965 INFO org.apache.hadoop.yarn.client.api.impl.YarnClientImpl - Submitted application application_1578233120766_0003</span><br><span class="line">2020-01-05 23:06:10,966 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Waiting for the cluster to be allocated</span><br><span class="line">2020-01-05 23:06:10,971 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Deploying cluster, current state ACCEPTED</span><br><span class="line">2020-01-05 23:06:18,402 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - YARN application has been deployed successfully.</span><br><span class="line">Starting execution of program</span><br><span class="line">Executing WordCount example with default input data set.</span><br><span class="line">Use --input to specify file input.</span><br><span class="line">Printing result to stdout. Use --output to specify output path.</span><br><span class="line">(a,5)</span><br><span class="line">(action,1)</span><br><span class="line">(after,1)</span><br><span class="line">(against,1)</span><br><span class="line">(all,2)</span><br><span class="line">(and,12)</span><br><span class="line">(arms,1)</span><br><span class="line">(arrows,1)</span><br><span class="line">(awry,1)</span><br><span class="line">(ay,1)</span><br><span class="line">(bare,1)</span><br><span class="line">(be,4)</span><br><span class="line">(bear,3)</span><br><span class="line">(bodkin,1)</span><br><span class="line">(bourn,1)</span><br><span class="line">(but,1)</span><br><span class="line">(by,2)</span><br><span class="line">(calamity,1)</span><br><span class="line">(cast,1)</span><br><span class="line">(coil,1)</span><br><span class="line">(come,1)</span><br><span class="line">(conscience,1)</span><br><span class="line">(consummation,1)</span><br><span class="line">(contumely,1)</span><br><span class="line">(country,1)</span><br><span class="line">(cowards,1)</span><br><span class="line">(currents,1)</span><br><span class="line">(d,4)</span><br><span class="line">(death,2)</span><br><span class="line">(delay,1)</span><br><span class="line">(despis,1)</span><br><span class="line">(devoutly,1)</span><br><span class="line">(die,2)</span><br><span class="line">(does,1)</span><br><span class="line">(dread,1)</span><br><span class="line">(dream,1)</span><br><span class="line">(dreams,1)</span><br><span class="line">(end,2)</span><br><span class="line">(enterprises,1)</span><br><span class="line">(er,1)</span><br><span class="line">(fair,1)</span><br><span class="line">(fardels,1)</span><br><span class="line">(flesh,1)</span><br><span class="line">(fly,1)</span><br><span class="line">(for,2)</span><br><span class="line">(fortune,1)</span><br><span class="line">(from,1)</span><br><span class="line">(give,1)</span><br><span class="line">(great,1)</span><br><span class="line">(grunt,1)</span><br><span class="line">(have,2)</span><br><span class="line">(he,1)</span><br><span class="line">(heartache,1)</span><br><span class="line">(heir,1)</span><br><span class="line">(himself,1)</span><br><span class="line">(his,1)</span><br><span class="line">(hue,1)</span><br><span class="line">(ills,1)</span><br><span class="line">(in,3)</span><br><span class="line">(insolence,1)</span><br><span class="line">(is,3)</span><br><span class="line">(know,1)</span><br><span class="line">(law,1)</span><br><span class="line">(life,2)</span><br><span class="line">(long,1)</span><br><span class="line">(lose,1)</span><br><span class="line">(love,1)</span><br><span class="line">(make,2)</span><br><span class="line">(makes,2)</span><br><span class="line">(man,1)</span><br><span class="line">(may,1)</span><br><span class="line">(merit,1)</span><br><span class="line">(might,1)</span><br><span class="line">(mind,1)</span><br><span class="line">(moment,1)</span><br><span class="line">(more,1)</span><br><span class="line">(mortal,1)</span><br><span class="line">(must,1)</span><br><span class="line">(my,1)</span><br><span class="line">(name,1)</span><br><span class="line">(native,1)</span><br><span class="line">(natural,1)</span><br><span class="line">(no,2)</span><br><span class="line">(nobler,1)</span><br><span class="line">(not,2)</span><br><span class="line">(now,1)</span><br><span class="line">(nymph,1)</span><br><span class="line">(o,1)</span><br><span class="line">(of,15)</span><br><span class="line">(off,1)</span><br><span class="line">(office,1)</span><br><span class="line">(ophelia,1)</span><br><span class="line">(opposing,1)</span><br><span class="line">(oppressor,1)</span><br><span class="line">(or,2)</span><br><span class="line">(orisons,1)</span><br><span class="line">(others,1)</span><br><span class="line">(outrageous,1)</span><br><span class="line">(pale,1)</span><br><span class="line">(pangs,1)</span><br><span class="line">(patient,1)</span><br><span class="line">(pause,1)</span><br><span class="line">(perchance,1)</span><br><span class="line">(pith,1)</span><br><span class="line">(proud,1)</span><br><span class="line">(puzzles,1)</span><br><span class="line">(question,1)</span><br><span class="line">(quietus,1)</span><br><span class="line">(rather,1)</span><br><span class="line">(regard,1)</span><br><span class="line">(remember,1)</span><br><span class="line">(resolution,1)</span><br><span class="line">(respect,1)</span><br><span class="line">(returns,1)</span><br><span class="line">(rub,1)</span><br><span class="line">(s,5)</span><br><span class="line">(say,1)</span><br><span class="line">(scorns,1)</span><br><span class="line">(sea,1)</span><br><span class="line">(shocks,1)</span><br><span class="line">(shuffled,1)</span><br><span class="line">(sicklied,1)</span><br><span class="line">(sins,1)</span><br><span class="line">(sleep,5)</span><br><span class="line">(slings,1)</span><br><span class="line">(so,1)</span><br><span class="line">(soft,1)</span><br><span class="line">(something,1)</span><br><span class="line">(spurns,1)</span><br><span class="line">(suffer,1)</span><br><span class="line">(sweat,1)</span><br><span class="line">(take,1)</span><br><span class="line">(takes,1)</span><br><span class="line">(than,1)</span><br><span class="line">(that,7)</span><br><span class="line">(the,22)</span><br><span class="line">(their,1)</span><br><span class="line">(them,1)</span><br><span class="line">(there,2)</span><br><span class="line">(these,1)</span><br><span class="line">(this,2)</span><br><span class="line">(those,1)</span><br><span class="line">(thought,1)</span><br><span class="line">(thousand,1)</span><br><span class="line">(thus,2)</span><br><span class="line">(thy,1)</span><br><span class="line">(time,1)</span><br><span class="line">(tis,2)</span><br><span class="line">(to,15)</span><br><span class="line">(traveller,1)</span><br><span class="line">(troubles,1)</span><br><span class="line">(turn,1)</span><br><span class="line">(under,1)</span><br><span class="line">(undiscover,1)</span><br><span class="line">(unworthy,1)</span><br><span class="line">(us,3)</span><br><span class="line">(we,4)</span><br><span class="line">(weary,1)</span><br><span class="line">(what,1)</span><br><span class="line">(when,2)</span><br><span class="line">(whether,1)</span><br><span class="line">(whips,1)</span><br><span class="line">(who,2)</span><br><span class="line">(whose,1)</span><br><span class="line">(will,1)</span><br><span class="line">(wish,1)</span><br><span class="line">(with,3)</span><br><span class="line">(would,2)</span><br><span class="line">(wrong,1)</span><br><span class="line">(you,1)</span><br><span class="line">Program execution finished</span><br><span class="line">Job with JobID 0af373b2d338811166328e7e1fca1fe1 has finished.</span><br><span class="line">Job Runtime: 10995 ms</span><br><span class="line">Accumulator Results: </span><br><span class="line">- fed80ecbe6107c17759a1be247abbdd4 (java.util.ArrayList) [170 elements]</span><br><span class="line"></span><br><span class="line">[double_happy@hadoop101 flink]$</span><br></pre></td></tr></table></figure></div>
<p><strong>运行一下 之前的local的 程序：</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">运行一下 之前的local的 程序：</span><br><span class="line">[double_happy@hadoop101 flink]$ ./bin/flink run \</span><br><span class="line">> --class com.sx.flink04.SocketWindowWordCount \</span><br><span class="line">> -m yarn-cluster \</span><br><span class="line">> /home/double_happy/lib/Flink-1.0.jar \</span><br><span class="line">> --port 9998 \</span><br><span class="line">> --host hadoop101</span><br><span class="line">2020-01-05 23:08:37,619 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 23:08:37,619 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - Found Yarn properties file under /tmp/.yarn-properties-double_happy.</span><br><span class="line">2020-01-05 23:08:38,076 INFO org.apache.hadoop.yarn.client.RMProxy - Connecting to ResourceManager at /0.0.0.0:8032</span><br><span class="line">2020-01-05 23:08:38,294 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 23:08:38,294 INFO org.apache.flink.yarn.cli.FlinkYarnSessionCli - No path for the flink jar passed. Using the location of class org.apache.flink.yarn.YarnClusterDescriptor to locate the jar</span><br><span class="line">2020-01-05 23:08:38,467 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Cluster specification: ClusterSpecification{masterMemoryMB=1024, taskManagerMemoryMB=1024, numberTaskManagers=1, slotsPerTaskManager=1}</span><br><span class="line">2020-01-05 23:08:39,079 WARN org.apache.flink.yarn.AbstractYarnClusterDescriptor - The configuration directory ('/home/double_happy/app/flink-1.9.1/conf') contains both LOG4J and Logback configuration files. Please delete or rename one of them.</span><br><span class="line">2020-01-05 23:08:40,769 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Submitting application master application_1578233120766_0004</span><br><span class="line">2020-01-05 23:08:41,006 INFO org.apache.hadoop.yarn.client.api.impl.YarnClientImpl - Submitted application application_1578233120766_0004</span><br><span class="line">2020-01-05 23:08:41,007 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Waiting for the cluster to be allocated</span><br><span class="line">2020-01-05 23:08:41,010 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - Deploying cluster, current state ACCEPTED</span><br><span class="line">2020-01-05 23:08:46,914 INFO org.apache.flink.yarn.AbstractYarnClusterDescriptor - YARN application has been deployed successfully.</span><br><span class="line">Starting execution of program</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/2020010523100919.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">输入数据：</span><br><span class="line">[double_happy@hadoop101 data]$ nc -lk 9998</span><br><span class="line">啊，a,a,a,v,b,c</span><br><span class="line">c,c,d,d,,b,b</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">查看结果：</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200105231116600.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">这是yarn的两种方式：</span><br><span class="line"></span><br><span class="line">我使用的是第二种方式 ：</span><br><span class="line"> 如果需要一个MySQL jar包该怎么传？</span><br><span class="line"> 我接触flink 时间很短 我身边同事也不知道 他们经常搞Flink的 竟然不知道？ 就知道打胖包？ 绝对是不可以的</span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-Flink03-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/01/05/Flink03-double-happy/">Flink03--double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/01/05/Flink03-double-happy/" class="article-date">
<time datetime="2020-01-05T15:47:23.000Z" itemprop="datePublished">2020-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="记录"><a href="#记录" class="headerlink" title="记录"></a>记录</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">写东西 还是要有通用的思想</span><br><span class="line"></span><br><span class="line">通用的！！！ 与底层的执行引擎是没关系的 Beam</span><br><span class="line"> Spark</span><br><span class="line"> 1.6</span><br><span class="line"> 2.x</span><br><span class="line"> Flink</span><br><span class="line"></span><br><span class="line"> 你的API上一定不能出现底层执行引擎的API</span><br><span class="line"></span><br><span class="line"> 适配 ==> Spark/Flink</span><br></pre></td></tr></table></figure></div>
<h2 id="注意"><a href="#注意" class="headerlink" title="注意"></a>注意</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br></pre></td><td class="code"><pre><span class="line">上一篇文章 ：</span><br><span class="line">MySQLSource：</span><br><span class="line">class MySQLSource extends RichSourceFunction[Student]{</span><br><span class="line"></span><br><span class="line"> var connection:Connection = _</span><br><span class="line"> var pstmt:PreparedStatement = _</span><br><span class="line"></span><br><span class="line"> // 在open方法中建立连接</span><br><span class="line"> override def open(parameters: Configuration): Unit = {</span><br><span class="line"> super.open(parameters)</span><br><span class="line"></span><br><span class="line"> connection = MySQLUtils.getConnection("hadoop101","3306","test","root", "wsx123$%^")</span><br><span class="line"> pstmt = connection.prepareStatement("select * from student")</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> // 释放</span><br><span class="line"> override def close(): Unit = {</span><br><span class="line"> super.close()</span><br><span class="line"> MySQLUtils.closeConnection(connection, pstmt)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Student]): Unit = {</span><br><span class="line"></span><br><span class="line"> println("~~~~run~~~~~~")</span><br><span class="line"> val rs = pstmt.executeQuery()</span><br><span class="line"> while(rs.next()){</span><br><span class="line"> val student = Student(rs.getInt("id"), rs.getString("name"), rs.getString("password"),rs.getInt("age"))</span><br><span class="line"> ctx.collect(student)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line">class MySQLSource extends RichSourceFunction[Student]</span><br><span class="line"></span><br><span class="line">RichSourceFunction：</span><br><span class="line">这个东西 并不是 上一篇文章的三个Function 中的一个 </span><br><span class="line"></span><br><span class="line">那么 RichSourceFunction 的并行度是多少？</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">源码：</span><br><span class="line">public abstract class RichSourceFunction<OUT> extends AbstractRichFunction implements SourceFunction<OUT> {</span><br><span class="line"></span><br><span class="line"> private static final long serialVersionUID = 1L;</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">我当时测试的时候 设置并行度为3的时候 报错的 </span><br><span class="line"></span><br><span class="line">源码里是继承SourceFunction的 </span><br><span class="line">所以并行度是1</span><br><span class="line"></span><br><span class="line">那么就有一个问题？</span><br><span class="line"> 有两个流：</span><br><span class="line"> </span><br><span class="line"> stream1 4并行度 emp表 log里 domain</span><br><span class="line"> stream2 1并行度 dept表 MySQL里 user_id</span><br><span class="line">这两个流进行 </span><br><span class="line">stream1.connect(stream2).map(x=>{})</span><br><span class="line"></span><br><span class="line">即：</span><br><span class="line"> logStream emp: deptno empno ename</span><br><span class="line"> mysqlStream dept: deptno dname</span><br><span class="line"> </span><br><span class="line"> outputStream empno,ename,deptno,dname</span><br><span class="line"> </span><br><span class="line"> 数据清洗/ETL</span><br><span class="line"> outputStream = stream1.connect(stream2).map(x=>{</span><br><span class="line"> ....</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line">那么connect会成功么？？</span><br><span class="line"> 4并行度的流 里面有写task 的deptno 关联 </span><br><span class="line"> 1并行度的流 拿取dname 是拿不到的 </span><br><span class="line">那么这个问题该怎么解决呢？之后再说</span><br><span class="line"></span><br><span class="line">先解决并行度为1的问题 下面</span><br><span class="line"> 变为：RichParallelSourceFunction</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">class ScalikeJDBCMySQLSource extends RichParallelSourceFunction[Student]{</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Student]): Unit = {</span><br><span class="line"> println("~~~run~~~~")</span><br><span class="line"> DBs.setupAll() // parse configuration file</span><br><span class="line"></span><br><span class="line"> DB.readOnly{ implicit session => {</span><br><span class="line"> SQL("select * from student").map(rs => {</span><br><span class="line"> val student = Student(rs.int("id"),rs.string("name"),rs.string("password"),rs.int("age"))</span><br><span class="line"> ctx.collect(student)</span><br><span class="line"> }).list().apply()</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> env.addSource(new ScalikeJDBCMySQLSource).setParallelism(2).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">~~~run~~~~</span><br><span class="line">~~~run~~~~</span><br><span class="line">6> Student(1,kairis,wsx111,17)</span><br><span class="line">1> Student(4,happy,11,48)</span><br><span class="line">8> Student(3,double,44,12)</span><br><span class="line">7> Student(2,dbh,11,90)</span><br><span class="line">7> Student(3,double,44,12)</span><br><span class="line">5> Student(1,kairis,wsx111,17)</span><br><span class="line">6> Student(2,dbh,11,90)</span><br><span class="line">8> Student(4,happy,11,48)</span><br><span class="line"></span><br><span class="line">2并行度 也就是结果会拿取两份</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">第二个问题：</span><br><span class="line">接着最开始流connect的问题 </span><br><span class="line"></span><br><span class="line">1.log流 的并行度是4 （100w条）</span><br><span class="line">那么 MySQL里的并行度是否有必要也是4呢？ （MySQl 1w条）</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line"> 是不是 没有这个必要呀 </span><br><span class="line"> 1个并行度能处理过来的 就可以了 </span><br><span class="line"> 但是1个并行度 也log流4个并行度 connect的时候还会出问题</span><br><span class="line">那么该怎么办呢？之后说</span><br></pre></td></tr></table></figure></div>
<h2 id="Data-Sinks"><a href="#Data-Sinks" class="headerlink" title="Data Sinks"></a>Data Sinks</h2><p><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/datastream_api.html#data-sinks" target="_blank" rel="noopener">Data Sinks</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">输出对比：</span><br><span class="line">Spark读写：外部数据源</span><br><span class="line"> spark.read.format("").option("",").load</span><br><span class="line"> spark.write.format("")....save</span><br><span class="line"></span><br><span class="line">Flink读写</span><br><span class="line"> addSource(new XXXSourceFunction)</span><br><span class="line"> addSink(new XXXSinkFunction)</span><br><span class="line"></span><br><span class="line"> 都是 可插拔的 Spark 自定义外部数据源 </span><br><span class="line">还是Spark好用呀</span><br></pre></td></tr></table></figure></div>
<h2 id="案例"><a href="#案例" class="headerlink" title="案例"></a>案例</h2><p><strong>Sink kafka</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line">Flink的角色是Producer：</span><br><span class="line"></span><br><span class="line">object SinkApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong).toString</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> val producer = new FlinkKafkaProducer[String](</span><br><span class="line"> "hadoop101:9092,hadoop101:9093,hadoop101:9094", // broker list</span><br><span class="line"> "double_happy_offset", // target topic</span><br><span class="line"> new SimpleStringSchema) // serialization schema</span><br><span class="line"></span><br><span class="line"> stream.addSink(producer) // 2Kafka</span><br><span class="line"> stream.print() // 2Local</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line">结果是：</span><br><span class="line">[double_happy@hadoop101 kafka]$ bin/kafka-console-consumer.sh \</span><br><span class="line">> --bootstrap-server hadoop101:9092,hadoop101:9093,hadoop101:9094 \</span><br><span class="line">> --topic double_happy_offset </span><br><span class="line">Access(201912120010,ruozedata.com,2000)</span><br><span class="line">Access(201912120010,ruozedata.com,4000)</span><br><span class="line">Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">Access(201912120010,zhibo8.com,5000)</span><br></pre></td></tr></table></figure></div>
<p><strong>Kafak 2 Kafka</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">object SinkApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val properties = new Properties()</span><br><span class="line"> properties.setProperty("bootstrap.servers", "hadoop101:9092,hadoop101:9093,hadoop101:9094")</span><br><span class="line"> properties.setProperty("group.id", "sxwang")</span><br><span class="line"> </span><br><span class="line"> val consumer = new FlinkKafkaConsumer[String]("double_happy_offset", new SimpleStringSchema(), properties)</span><br><span class="line"> </span><br><span class="line"> val stream = env.addSource(consumer)</span><br><span class="line"></span><br><span class="line"> // TODO... Kafka2Kafka double_happy_offset ==> double_happy_offset_test</span><br><span class="line"> </span><br><span class="line"> val producer = new FlinkKafkaProducer[String](</span><br><span class="line"> "hadoop101:9092,hadoop101:9093,hadoop101:9094", // broker list</span><br><span class="line"> "double_happy_offset_test", // target topic</span><br><span class="line"> new SimpleStringSchema) // serialization schema</span><br><span class="line"></span><br><span class="line"> stream.addSink(producer) // 2Kafka</span><br><span class="line"> stream.print() // 2Local</span><br><span class="line"> </span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">[double_happy@hadoop101 kafka]$ bin/kafka-console-consumer.sh \</span><br><span class="line">> --bootstrap-server hadoop101:9092,hadoop101:9093,hadoop101:9094 \</span><br><span class="line">> --topic double_happy_offset_test </span><br><span class="line">Access(201912120010,ruozedata.com,2000)</span><br><span class="line">Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">e</span><br><span class="line">c</span><br><span class="line">Access(201912120010,ruozedata.com,4000)</span><br><span class="line">b</span><br><span class="line">e</span><br><span class="line">Access(201912120010,zhibo8.com,5000)</span><br><span class="line">c</span><br><span class="line">f</span><br></pre></td></tr></table></figure></div>
<p><strong>Sink MySQL</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">/**</span><br><span class="line"> * A {@link org.apache.flink.api.common.functions.RichFunction} version of {@link SinkFunction}.</span><br><span class="line"> */</span><br><span class="line">@Public</span><br><span class="line">public abstract class RichSinkFunction<IN> extends AbstractRichFunction implements SinkFunction<IN> {</span><br><span class="line"></span><br><span class="line"> private static final long serialVersionUID = 1L;</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> * Interface for implementing user defined sink functionality.</span><br><span class="line"> *</span><br><span class="line"> * @param <IN> Input type parameter.</span><br><span class="line"> */</span><br><span class="line">@Public</span><br><span class="line">public interface SinkFunction<IN> extends Function, Serializable {</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">所以</span><br><span class="line">IN： 表示 Input type parameter. </span><br><span class="line"></span><br><span class="line">表示sink到的 MySQL表里的 字段类型</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br></pre></td><td class="code"><pre><span class="line">class DoubleHappyMySQLSink extends RichSinkFunction[(String,Int)]{</span><br><span class="line"></span><br><span class="line"> var connection:Connection = _</span><br><span class="line"> var insertPstmt:PreparedStatement = _</span><br><span class="line"> var updatePstmt:PreparedStatement = _</span><br><span class="line"></span><br><span class="line"> // 打开connection等</span><br><span class="line"> override def open(parameters: Configuration): Unit = {</span><br><span class="line"> super.open(parameters)</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 这块与Spark不一样的 Spark是一个批次往下写 </span><br><span class="line"> * </span><br><span class="line"> * 而Flink是 来一条就往下写 </span><br><span class="line"> * </span><br><span class="line"> * 所以 :</span><br><span class="line"> * 这块创建两个 PreparedStatement 也可以创建一个 需要创建表的时候指定 key 即可 </span><br><span class="line"> * </span><br><span class="line"> * 创建 两个简单些 ： 有就更新 没有就插入 </span><br><span class="line"> */</span><br><span class="line"> connection = MySQLUtils.getConnection()</span><br><span class="line"> insertPstmt = connection.prepareStatement("insert into domain(domain,traffic) values (?,?)")</span><br><span class="line"> updatePstmt = connection.prepareStatement("update domain set traffic=? where domain=?")</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 写数据</span><br><span class="line"> */</span><br><span class="line"> override def invoke(value: (String, Int), context: SinkFunction.Context[_]): Unit = {</span><br><span class="line"> // TODO insert update</span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * 这块并没有写错哦 要看 你的sql语句的 ？ 走的哦 </span><br><span class="line"> * </span><br><span class="line"> * 先更新 有就更新 无就插入</span><br><span class="line"> */</span><br><span class="line"> updatePstmt.setInt(1, value._2)</span><br><span class="line"> updatePstmt.setString(2, value._1)</span><br><span class="line"> updatePstmt.execute()</span><br><span class="line"></span><br><span class="line"> if(updatePstmt.getUpdateCount == 0) {</span><br><span class="line"> insertPstmt.setString(1, value._1)</span><br><span class="line"> insertPstmt.setInt(2, value._2)</span><br><span class="line"> insertPstmt.execute()</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> // 释放资源</span><br><span class="line"> override def close(): Unit = {</span><br><span class="line"> super.close()</span><br><span class="line"> if(insertPstmt != null) insertPstmt.close()</span><br><span class="line"> if(updatePstmt != null) updatePstmt.close()</span><br><span class="line"> if(connection != null) connection.close()</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">这代码不严谨的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br></pre></td><td class="code"><pre><span class="line">object SinkApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> (splits(1), splits(2).toInt)</span><br><span class="line"> }).keyBy(0).sum(1)</span><br><span class="line"></span><br><span class="line"> stream.addSink(new DoubleHappyMySQLSink)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是： 我执行了两遍Flink程序 发现 结果幂等性 ok</span><br><span class="line">mysql> select * from domain;</span><br><span class="line">+---------------+---------+</span><br><span class="line">| domain | traffic |</span><br><span class="line">+---------------+---------+</span><br><span class="line">| dongqiudi.com | 7000 |</span><br><span class="line">| ruozedata.com | 6000 |</span><br><span class="line">| zhibo8.com | 5000 |</span><br><span class="line">+---------------+---------+</span><br><span class="line">3 rows in set (0.00 sec)</span><br><span class="line"></span><br><span class="line">mysql> select * from domain;</span><br><span class="line">+---------------+---------+</span><br><span class="line">| domain | traffic |</span><br><span class="line">+---------------+---------+</span><br><span class="line">| dongqiudi.com | 7000 |</span><br><span class="line">| ruozedata.com | 6000 |</span><br><span class="line">| zhibo8.com | 5000 |</span><br><span class="line">+---------------+---------+</span><br><span class="line">3 rows in set (0.00 sec)</span><br><span class="line"></span><br><span class="line">mysql></span><br></pre></td></tr></table></figure></div>
<h2 id="Debugging"><a href="#Debugging" class="headerlink" title="Debugging ***"></a>Debugging ***</h2><p>调试的手段***</p>
<p><strong>Iterator Data Sink</strong><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/datastream_api.html#iterator-data-sink" target="_blank" rel="noopener">Iterator Data Sink</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> 你调试的时候 ：</span><br><span class="line"> 1.不需要接kafka 使用 自定义Source 就可以 eg：RichParallelSourceFunction</span><br><span class="line"> 2.Sink的时候 也不需要Sink 到 存储的地方去</span><br><span class="line"> eg：kudu 、Hbase等</span><br><span class="line"> 这里使用 Iterator Data Sink</span><br><span class="line">这个真的很重要：</span><br><span class="line"> 我封闭开发的时候 字段154个 Sink到kudu里面 </span><br><span class="line">如果使用 提供的 connecer 154个字段 你什么时候能写完 </span><br><span class="line"> 一定是要写成 自动解析的 最好 </span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 刚好 我的业务场景 需要这个 Iterator Data Sink</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">官网：</span><br><span class="line">import org.apache.flink.streaming.experimental.DataStreamUtils</span><br><span class="line">import scala.collection.JavaConverters.asScalaIteratorConverter</span><br><span class="line"></span><br><span class="line">val myResult: DataStream[(String, Int)] = ...</span><br><span class="line">val myOutput: Iterator[(String, Int)] = DataStreamUtils.collect(myResult.javaStream).asScala</span><br></pre></td></tr></table></figure></div>
<p>案例：</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"></span><br><span class="line"> val data: DataStream[Domain.Access] = env.addSource(new AccessSource03).setParallelism(1)</span><br><span class="line"></span><br><span class="line"> import scala.collection.JavaConverters.asScalaIteratorConverter</span><br><span class="line"> val output: Iterator[Domain.Access] = DataStreamUtils.collect(data.javaStream).asScala</span><br><span class="line"> </span><br><span class="line"> output.take(1).foreach(println(_))</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">Access(1577975577015,zhibo8.cc,78)</span><br></pre></td></tr></table></figure></div>
<h2 id="Sink-redis"><a href="#Sink-redis" class="headerlink" title="Sink redis"></a>Sink redis</h2><p><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/connectors/" target="_blank" rel="noopener">connector</a><br><img src="https://img-blog.csdnimg.cn/20200102225345666.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line">pom 选择的 版本;</span><br><span class="line"></span><br><span class="line"><dependency></span><br><span class="line"> <groupId>org.apache.bahir</groupId></span><br><span class="line"> <artifactId>flink-connector-redis_2.11</artifactId></span><br><span class="line"> <version>1.1-SNAPSHOT</version></span><br><span class="line"></dependency></span><br><span class="line"></span><br><span class="line">这个版本可能下载不到 ：</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20200102225516772.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">所以下载：</span><br><span class="line"><!-- https://mvnrepository.com/artifact/org.apache.bahir/flink-connector-redis --></span><br><span class="line"><dependency></span><br><span class="line"> <groupId>org.apache.bahir</groupId></span><br><span class="line"> <artifactId>flink-connector-redis_2.11</artifactId></span><br><span class="line"> <version>1.0</version></span><br><span class="line"></dependency></span><br></pre></td></tr></table></figure></div>
<p><a href="https://bahir.apache.org/docs/flink/current/flink-streaming-redis/" target="_blank" rel="noopener">Flink Redis Connector</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">class DoubleHappyRedisSink extends RedisMapper[(String,Int)]{</span><br><span class="line"> override def getCommandDescription: RedisCommandDescription = {</span><br><span class="line"> new RedisCommandDescription(RedisCommand.HSET, "double_happy_traffic")</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def getValueFromData(data: (String, Int)): String = {</span><br><span class="line"> data._2 + ""</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def getKeyFromData(data: (String, Int)): String = {</span><br><span class="line"> data._1</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line">object SinkApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> (splits(1), splits(2).toInt)</span><br><span class="line"> }).keyBy(0).sum(1)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> val conf = new FlinkJedisPoolConfig.Builder().setHost("hadoop101").build()</span><br><span class="line"></span><br><span class="line"> stream.addSink(new RedisSink[(String, Int)](conf, new DoubleHappyRedisSink))</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line">结果是：</span><br><span class="line">hadoop101:6379> HGETALL double_happy_traffic</span><br><span class="line">1) "ruozedata.com"</span><br><span class="line">2) "6000"</span><br><span class="line">3) "dongqiudi.com"</span><br><span class="line">4) "7000"</span><br><span class="line">5) "zhibo8.com"</span><br><span class="line">6) "5000"</span><br><span class="line">hadoop101:6379> </span><br><span class="line"></span><br><span class="line">与前面的MySQL 结果是一样的</span><br></pre></td></tr></table></figure></div>
<h2 id="解决前面的问题"><a href="#解决前面的问题" class="headerlink" title="***解决前面的问题"></a>***解决前面的问题</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"></span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-Flink02-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/01/05/Flink02-double-happy/">Flink02--double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/01/05/Flink02-double-happy/" class="article-date">
<time datetime="2020-01-05T15:46:29.000Z" itemprop="datePublished">2020-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="概述"><a href="#概述" class="headerlink" title="概述"></a>概述</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">Function</span><br><span class="line"> map ==> MapFunction</span><br><span class="line"> filter ==> FilterFunction</span><br><span class="line"> xxx ==> XxxFunction</span><br><span class="line"> RichXxxFunction *****</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">SourceFunction non-parallel 1</span><br><span class="line">ParallelSourceFunction</span><br><span class="line">RichParallelSourceFunction *****</span><br></pre></td></tr></table></figure></div>
<p><strong>//测试使用</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"> env.fromCollection(List(</span><br><span class="line"> Access(201912120010L, "ruozedata.com", 2000),</span><br><span class="line"> Access(201912120011L, "ruozedata.com", 3000)</span><br><span class="line"> )</span><br><span class="line"> ).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">6> Access(201912120011,ruozedata.com,3000)</span><br><span class="line">5> Access(201912120010,ruozedata.com,2000)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">/**</span><br><span class="line"> * Creates a DataStream that contains the given elements. The elements must all be of the</span><br><span class="line"> * same type.</span><br><span class="line"> *</span><br><span class="line"> * Note that this operation will result in a non-parallel data source, i.e. a data source with</span><br><span class="line"> * a parallelism of one.</span><br><span class="line"> */</span><br><span class="line"> def fromElements[T: TypeInformation](data: T*): DataStream[T] = {</span><br><span class="line"> fromCollection(data)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 前面的泛型 如果加上 会类型限定的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"> env.fromElements(1, 2L, 3D, 4F, "5").print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">1> 4.0</span><br><span class="line">8> 3.0</span><br><span class="line">2> 5</span><br><span class="line">7> 2</span><br><span class="line">6> 1</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> 如果加上泛型 会类型限定的</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191225232021656.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><strong>Custom:</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">env.addSource()</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">1.SourceFunction </span><br><span class="line"> cancel()</span><br><span class="line"> run()</span><br><span class="line">注意：</span><br><span class="line"> 1.自定义数据源</span><br><span class="line"> 继承SourceFunction 要传一个泛型</span><br><span class="line"> 2.实现 下面这两个方法</span><br><span class="line"> cancel()</span><br><span class="line"> 用于关闭某个东西</span><br><span class="line"> run() 里面有ctx 上下文</span><br><span class="line"> 用于产生数据的 </span><br><span class="line"> 因为这是SourceFunction 是数据源</span><br><span class="line"> 可以利用ctx 产生数据的</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> SourceFunction 是没有并行度可言的</span><br></pre></td></tr></table></figure></div>
<p><strong>这个很重要用于测试产生数据</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line">package com.sx.flink02</span><br><span class="line"></span><br><span class="line">import com.sx.bean.Domain.Access</span><br><span class="line">import org.apache.flink.streaming.api.functions.source.SourceFunction</span><br><span class="line"></span><br><span class="line">import scala.util.Random</span><br><span class="line"></span><br><span class="line">class AccessSource extends SourceFunction[Access]{</span><br><span class="line"></span><br><span class="line"> var running = true</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"> running = false</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Access]): Unit = {</span><br><span class="line"> val random = new Random()</span><br><span class="line"> val domains = Array("ruozedata.com","zhibo8.cc","dongqiudi.com")</span><br><span class="line"></span><br><span class="line"> while(running) {</span><br><span class="line"> val timestamp = System.currentTimeMillis()</span><br><span class="line"> 1.to(10).map(x => {</span><br><span class="line"> ctx.collect(Access(timestamp, domains(random.nextInt(domains.length)), random.nextInt(1000)+x))</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> // 休息下 每个5s</span><br><span class="line"> Thread.sleep(5000)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"> env.addSource(new AccessSource).print()</span><br><span class="line"> </span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">7> Access(1577502637884,zhibo8.cc,991)</span><br><span class="line">2> Access(1577502637884,ruozedata.com,252)</span><br><span class="line">5> Access(1577502637884,dongqiudi.com,428)</span><br><span class="line">8> Access(1577502637884,dongqiudi.com,565)</span><br><span class="line">4> Access(1577502637884,ruozedata.com,140)</span><br><span class="line">6> Access(1577502637884,ruozedata.com,718)</span><br><span class="line">1> Access(1577502637884,dongqiudi.com,66)</span><br><span class="line">3> Access(1577502637884,dongqiudi.com,475)</span><br><span class="line">4> Access(1577502637884,zhibo8.cc,694)</span><br><span class="line">3> Access(1577502637884,zhibo8.cc,972)</span><br><span class="line"></span><br><span class="line">这就是每个5s发送10条数据</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line">测试:</span><br><span class="line">SourceFunction 是没有并行度可言的 </span><br><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"></span><br><span class="line"> env.addSource(new AccessSource).setParallelism(3).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line">结果：</span><br><span class="line">Exception in thread "main" java.lang.IllegalArgumentException: Source: 1 is not a parallel source</span><br><span class="line"> at org.apache.flink.streaming.api.datastream.DataStreamSource.setParallelism(DataStreamSource.java:55)</span><br><span class="line"> at org.apache.flink.streaming.api.datastream.DataStreamSource.setParallelism(DataStreamSource.java:31)</span><br><span class="line"> at org.apache.flink.streaming.api.scala.DataStream.setParallelism(DataStream.scala:130)</span><br><span class="line"> at com.sx.flink02.SourceApp$.main(SourceApp.scala:15)</span><br><span class="line"> at com.sx.flink02.SourceApp.main(SourceApp.scala)</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">Source: 1 is not a parallel source</span><br><span class="line"></span><br><span class="line">查看源码：</span><br><span class="line">debug查看</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/201912281116240.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><img src="https://img-blog.csdnimg.cn/20191228111651840.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">所以在大数据场景下 没有并行度 这个就不能用</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br></pre></td><td class="code"><pre><span class="line">2.ParallelSourceFunction 带并行度的 </span><br><span class="line"></span><br><span class="line">class AccessSource02 extends ParallelSourceFunction[Access]{</span><br><span class="line"></span><br><span class="line"> var running = true</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"> running = false</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Access]): Unit = {</span><br><span class="line"> val random = new Random()</span><br><span class="line"> val domains = Array("ruozedata.com","zhibo8.cc","dongqiudi.com")</span><br><span class="line"></span><br><span class="line"> while(running) {</span><br><span class="line"> val timestamp = System.currentTimeMillis()</span><br><span class="line"> 1.to(10).map(x => {</span><br><span class="line"> ctx.collect(Access(timestamp, domains(random.nextInt(domains.length)), random.nextInt(1000)+x))</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> // 休息下</span><br><span class="line"> Thread.sleep(5000)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">代码是一样的 就换个继承</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> //测试使用</span><br><span class="line"> env.addSource(new AccessSource02).setParallelism(3).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">6> Access(1577503201868,dongqiudi.com,704)</span><br><span class="line">8> Access(1577503201868,zhibo8.cc,562)</span><br><span class="line">2> Access(1577503201868,dongqiudi.com,526)</span><br><span class="line">7> Access(1577503201868,ruozedata.com,952)</span><br><span class="line">3> Access(1577503201868,ruozedata.com,525)</span><br><span class="line">2> Access(1577503201868,dongqiudi.com,728)</span><br><span class="line">7> Access(1577503201868,dongqiudi.com,185)</span><br><span class="line">5> Access(1577503201868,ruozedata.com,737)</span><br><span class="line">2> Access(1577503201868,zhibo8.cc,47)</span><br><span class="line">1> Access(1577503201868,dongqiudi.com,77)</span><br><span class="line">4> Access(1577503201868,ruozedata.com,275)</span><br><span class="line">5> Access(1577503201868,ruozedata.com,47)</span><br><span class="line">7> Access(1577503201868,dongqiudi.com,289)</span><br><span class="line">3> Access(1577503201868,dongqiudi.com,934)</span><br><span class="line">8> Access(1577503201868,ruozedata.com,69)</span><br><span class="line">6> Access(1577503201868,dongqiudi.com,123)</span><br><span class="line">4> Access(1577503201868,zhibo8.cc,719)</span><br><span class="line">1> Access(1577503201868,zhibo8.cc,254)</span><br><span class="line">6> Access(1577503201868,zhibo8.cc,1002)</span><br><span class="line">3> Access(1577503201868,zhibo8.cc,358)</span><br><span class="line">7> Access(1577503201868,dongqiudi.com,700)</span><br><span class="line">5> Access(1577503201868,dongqiudi.com,692)</span><br><span class="line">6> Access(1577503201868,ruozedata.com,408)</span><br><span class="line">5> Access(1577503201868,ruozedata.com,986)</span><br><span class="line">1> Access(1577503201868,ruozedata.com,284)</span><br><span class="line">4> Access(1577503201868,zhibo8.cc,900)</span><br><span class="line">5> Access(1577503201868,ruozedata.com,343)</span><br><span class="line">8> Access(1577503201868,ruozedata.com,84)</span><br><span class="line">4> Access(1577503201868,zhibo8.cc,967)</span><br><span class="line">8> Access(1577503201868,ruozedata.com,14)</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">这就是每个5s发送10条数据 但是并行度是3的情况下 </span><br><span class="line">就是 每个5s 发送 10*3 = 30 条数据</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">3.RichParallelSourceFunction ****</span><br><span class="line">注意：</span><br><span class="line"> 和上面的核心代码一样</span><br><span class="line"> 但是 这个Rich里面</span><br><span class="line"></span><br><span class="line">/**</span><br><span class="line"> * Base class for implementing a parallel data source. Upon execution, the runtime will</span><br><span class="line"> * execute as many parallel instances of this function as configured parallelism</span><br><span class="line"> * of the source.</span><br><span class="line"> *</span><br><span class="line"> * <p>The data source has access to context information (such as the number of parallel</span><br><span class="line"> * instances of the source, and which parallel instance the current instance is)</span><br><span class="line"> * via {@link #getRuntimeContext()}. It also provides additional life-cycle methods</span><br><span class="line"> * ({@link #open(org.apache.flink.configuration.Configuration)} and {@link #close()}.</p></span><br><span class="line"> *</span><br><span class="line"> * @param <OUT> The type of the records produced by this source.</span><br><span class="line"> */</span><br><span class="line">@Public</span><br><span class="line">public abstract class RichParallelSourceFunction<OUT> extends AbstractRichFunction</span><br><span class="line"> implements ParallelSourceFunction<OUT> {</span><br><span class="line"></span><br><span class="line"> private static final long serialVersionUID = 1L;</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">1. extends AbstractRichFunction </span><br><span class="line">而AbstractRichFunction </span><br><span class="line">是有生命周期的方法</span><br><span class="line"> open()</span><br><span class="line"> close()</span><br><span class="line"></span><br><span class="line">所以：</span><br><span class="line"> 如果做一些</span><br><span class="line"> 1.文件系统 </span><br><span class="line"> 2.初始化</span><br><span class="line"> 3.io</span><br><span class="line"> 4.mysql 等等</span><br><span class="line">就得在open里和close里面获取连接 和关闭连接</span><br><span class="line"></span><br><span class="line">因为：</span><br><span class="line"> open() close() 一个task会执行一次 （前面文章有例子）task就是并行度</span><br><span class="line"> 所以要使用这个Function </span><br><span class="line"> 生命周期是可以控制的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br></pre></td><td class="code"><pre><span class="line">class AccessSource03 extends RichParallelSourceFunction[Access]{</span><br><span class="line"></span><br><span class="line"> var running = true</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"> running = false</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def open(parameters: Configuration): Unit = {</span><br><span class="line"> super.open(parameters)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def close(): Unit = {</span><br><span class="line"> super.close()</span><br><span class="line"> }</span><br><span class="line"> </span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Access]): Unit = {</span><br><span class="line"> val random = new Random()</span><br><span class="line"> val domains = Array("ruozedata.com","zhibo8.cc","dongqiudi.com")</span><br><span class="line"></span><br><span class="line"> while(running) {</span><br><span class="line"> val timestamp = System.currentTimeMillis()</span><br><span class="line"> 1.to(10).map(x => {</span><br><span class="line"> ctx.collect(Access(timestamp, domains(random.nextInt(domains.length)), random.nextInt(1000)+x))</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> // 休息下</span><br><span class="line"> Thread.sleep(5000)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">上面就是 Source的最基本的使用</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">那么现在有一个需求：</span><br><span class="line"> 连接MySQL </span><br><span class="line"> 查看官网 没有MySQL的Connector</span><br><span class="line"></span><br><span class="line">那么只能去github上找 或者 自己写一个Source即可</span><br><span class="line"></span><br><span class="line">需求就是：</span><br><span class="line"> 把MySQL数据读取进来 </span><br><span class="line"></span><br><span class="line">操作MySQL pom里要加入MySQL依赖的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">MySQL里面的数据：</span><br><span class="line">mysql> select * from student;</span><br><span class="line">+----+--------+----------+-----+</span><br><span class="line">| id | name | password | age |</span><br><span class="line">+----+--------+----------+-----+</span><br><span class="line">| 1 | kairis | wsx111 | 17 |</span><br><span class="line">| 2 | dbh | 11 | 90 |</span><br><span class="line">| 3 | double | 44 | 12 |</span><br><span class="line">| 4 | happy | 11 | 48 |</span><br><span class="line">+----+--------+----------+-----+</span><br><span class="line">4 rows in set (0.00 sec)</span><br><span class="line"></span><br><span class="line">mysql></span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line">class MySQLSource extends RichSourceFunction[Student]{</span><br><span class="line"> var connection:Connection = _</span><br><span class="line"> var pstmt:PreparedStatement = _</span><br><span class="line"> // 在open方法中建立连接</span><br><span class="line"> override def open(parameters: Configuration): Unit = {</span><br><span class="line"> super.open(parameters)</span><br><span class="line"> connection = MySQLUtils.getConnection()</span><br><span class="line"> pstmt = connection.prepareStatement("select * from student")</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> // 释放</span><br><span class="line"> override def close(): Unit = {</span><br><span class="line"> super.close()</span><br><span class="line"> MySQLUtils.closeConnection(connection, pstmt)</span><br><span class="line"> }</span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Student]): Unit = {</span><br><span class="line"> val rs = pstmt.executeQuery()</span><br><span class="line"> while(rs.next()){</span><br><span class="line"> val student = Student(rs.getInt("id"), rs.getString("name"), rs.getString("password"),rs.getInt("age"))</span><br><span class="line"> ctx.collect(student)</span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line"> 连接MySQL 生产上是有并行度的 </span><br><span class="line"> 你不能在run里面建立连接 为什么呢？什么会触发run呢？主要的逻辑会触发run </span><br><span class="line"> 这就类似Spark里面的 foreach 和foreachPartiion</span><br><span class="line">得借助于 open方法：</span><br><span class="line"> 去拿到连接 这个是每个task会执行一次</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.addSource(new MySQLSource).print()</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">~~~~run~~~~~~</span><br><span class="line">5> Student(4,happy,11,48)</span><br><span class="line">4> Student(3,double,44,12)</span><br><span class="line">3> Student(2,dbh,11,90)</span><br><span class="line">2> Student(1,kairis,wsx111,17)</span><br><span class="line"></span><br><span class="line">env.addSource(new MySQLSource).setParallelism(3).print()</span><br><span class="line">会报错的：</span><br><span class="line">Exception in thread "main" java.lang.IllegalArgumentException: Source: 1 is not a parallel source</span><br><span class="line">为什么呢？</span><br><span class="line">但是这种方式 太low了 </span><br><span class="line"></span><br><span class="line"></span><br><span class="line">使用scalikjdbc： 这种方式更优雅些</span><br><span class="line">在配置文件里可以配置连接池</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre></td><td class="code"><pre><span class="line">class ScalikeJDBCMySQLSource extends RichSourceFunction[Student]{</span><br><span class="line"></span><br><span class="line"> override def cancel(): Unit = {</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def run(ctx: SourceFunction.SourceContext[Student]): Unit = {</span><br><span class="line"> println("~~~run~~~~")</span><br><span class="line"> DBs.setupAll() // parse configuration file</span><br><span class="line"></span><br><span class="line"> DB.readOnly{ implicit session => {</span><br><span class="line"> SQL("select * from student").map(rs => {</span><br><span class="line"> val student = Student(rs.int("id"),rs.string("name"),rs.string("password"),rs.int("age"))</span><br><span class="line"> ctx.collect(student)</span><br><span class="line"> }).list().apply()</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">~~~run~~~~</span><br><span class="line">7> Student(4,happy,11,48)</span><br><span class="line">5> Student(2,dbh,11,90)</span><br><span class="line">4> Student(1,kairis,wsx111,17)</span><br><span class="line">6> Student(3,double,44,12)</span><br></pre></td></tr></table></figure></div>
<p><strong>读取kafka的数据</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">注意：</span><br><span class="line">不同的kafka版本 Flink 使用的api不同</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191228122121721.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">但是在 0.11之后就统一了</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191228122438764.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">我的kafka是 2.2.1系列的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">所以我使用的 pom是：</span><br><span class="line">我的flink 是 1.9.0的</span><br><span class="line"><dependency></span><br><span class="line"> <groupId>org.apache.flink</groupId></span><br><span class="line"> <artifactId>flink-connector-kafka_2.11</artifactId></span><br><span class="line"> <version>1.9.0</version></span><br><span class="line"></dependency></span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">Flink’s Kafka consumer is</span><br><span class="line"> called FlinkKafkaConsumer08 (or 09 for</span><br><span class="line"> Kafka 0.9.0.x versions, etc. </span><br><span class="line"> or </span><br><span class="line"> just FlinkKafkaConsumer for Kafka >= 1.0.0 versions).</span><br><span class="line"> It provides access to one or more Kafka topics.</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">Source：</span><br><span class="line"> Flink作为Kafka的消费者</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre></td><td class="code"><pre><span class="line">/**</span><br><span class="line"> * Creates a new Kafka streaming source consumer.</span><br><span class="line"> *</span><br><span class="line"> * @param topic The name of the topic that should be consumed.</span><br><span class="line"> * @param valueDeserializer The de-/serializer used to convert between Kafka's byte messages and Flink's objects.</span><br><span class="line"> * @param props</span><br><span class="line"> */</span><br><span class="line"> public FlinkKafkaConsumer(String topic, DeserializationSchema<T> valueDeserializer, Properties props) {</span><br><span class="line"> this(Collections.singletonList(topic), valueDeserializer, props);</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">1.topic</span><br><span class="line">2.DeserializationSchema 接收数据 所以是反序列化器 消费数据 所以是反序列化的</span><br><span class="line">3.Properties</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 kafka]$ jps</span><br><span class="line">6544 Kafka</span><br><span class="line">7092 Kafka</span><br><span class="line">7892 Jps</span><br><span class="line">5959 QuorumPeerMain</span><br><span class="line">7723 Kafka</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre></td><td class="code"><pre><span class="line">先开启：</span><br><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val properties = new Properties()</span><br><span class="line"> properties.setProperty("bootstrap.servers", "hadoop101:9092,hadoop101:9093,hadoop101:9094")</span><br><span class="line"> properties.setProperty("group.id", "sxwang")</span><br><span class="line"></span><br><span class="line"> val consumer = new FlinkKafkaConsumer[String]("double_happy_offset", new SimpleStringSchema(), properties)</span><br><span class="line"></span><br><span class="line"> env.addSource(consumer).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line">结果：</span><br><span class="line">1> b</span><br><span class="line">2> d</span><br><span class="line">3> d</span><br><span class="line">1> c</span><br><span class="line">2> f</span><br><span class="line">3> b</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">往kafka发送数据：</span><br><span class="line">object DataGenerator {</span><br><span class="line"></span><br><span class="line"> private val logger: Logger = LoggerFactory.getLogger(DataGenerator.getClass)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"></span><br><span class="line"> val props = new Properties()</span><br><span class="line"> props.put("bootstrap.servers", "hadoop101:9092,hadoop101:9093,hadoop101:9094")</span><br><span class="line"> props.put("acks", "all")</span><br><span class="line"> props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer")</span><br><span class="line"> props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer")</span><br><span class="line"> val producer = new KafkaProducer[String, String](props)</span><br><span class="line"></span><br><span class="line"> for (i <- 0 to 5) {</span><br><span class="line"> Thread.sleep(100)</span><br><span class="line"></span><br><span class="line"> //拿一个abcdef</span><br><span class="line"> val word: String = String.valueOf((new Random().nextInt(6) + 'a').toChar)</span><br><span class="line"> val part = i % 3 //发到哪个分区 因为是三个分区</span><br><span class="line"></span><br><span class="line"> logger.error("word : {}", word)</span><br><span class="line"></span><br><span class="line"> val record = producer.send(new ProducerRecord[String, String]("double_happy_offset", part, "",word))</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> producer.close()</span><br><span class="line"> println("double_happy 数据产生完毕..........")</span><br><span class="line"> }</span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">Flink里面的kafka的offset 非常 好管理的 </span><br><span class="line">Spark里面的 是批次 每个批次的偏移量的管理 </span><br><span class="line">Flink是一条数据 进来的 一条数据进来的 </span><br><span class="line">控制偏移量就是api的使用 很简单</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">val myConsumer = new FlinkKafkaConsumer08[String](...)</span><br><span class="line">myConsumer.setStartFromEarliest() // start from the earliest record possible</span><br><span class="line">myConsumer.setStartFromLatest() // start from the latest record</span><br><span class="line">myConsumer.setStartFromTimestamp(...) // start from specified epoch timestamp (milliseconds)</span><br><span class="line">myConsumer.setStartFromGroupOffsets() // the default behaviour</span><br><span class="line"></span><br><span class="line">val stream = env.addSource(myConsumer)</span><br><span class="line"></span><br><span class="line">这是官网上的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">那么先讲使用 里面的状态和checkpoint 容错等 后续写入文章</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br></pre></td><td class="code"><pre><span class="line">object SourceApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val properties = new Properties()</span><br><span class="line"> properties.setProperty("bootstrap.servers", "hadoop101:9092,hadoop101:9093,hadoop101:9094")</span><br><span class="line"> properties.setProperty("group.id", "sxwang")</span><br><span class="line"></span><br><span class="line"> val consumer = new FlinkKafkaConsumer[String]("double_happy_offset", new SimpleStringSchema(), properties)</span><br><span class="line"></span><br><span class="line"> consumer.setStartFromEarliest()</span><br><span class="line"></span><br><span class="line"> env.addSource(consumer).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">1> a</span><br><span class="line">2> f</span><br><span class="line">3> b</span><br><span class="line">2> f</span><br><span class="line">1> b</span><br><span class="line">2> d</span><br><span class="line">3> e</span><br><span class="line">2> f</span><br><span class="line">1> c</span><br><span class="line">3> d</span><br><span class="line">3> b</span><br><span class="line"></span><br><span class="line">看 这就把我们前面写入kafka的数据读出来了</span><br><span class="line"></span><br><span class="line">所以Flink里的偏移量很简单 底层管理的很好</span><br><span class="line"></span><br><span class="line">问题：</span><br><span class="line"> 我能不能指定 一个分区 分区里面的 offset呢？</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191228125529931.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/stream/operators/index.html#datastream-transformations" target="_blank" rel="noopener">DataStream Transformations</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">和Spark大部分都差不多 </span><br><span class="line">下面说一下 Spark里没有的</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191228130532217.png" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> stream.keyBy(_.domain).sum("traffic").print("sum")</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line">结果是：</span><br><span class="line">sum:5> Access(201912120010,ruozedata.com,2000)</span><br><span class="line">sum:6> Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">sum:6> Access(201912120010,zhibo8.com,5000)</span><br><span class="line">sum:5> Access(201912120010,ruozedata.com,6000)</span><br><span class="line">sum:6> Access(201912120010,dongqiudi.com,7000)</span><br><span class="line"></span><br><span class="line">为什么结果是这样子的：</span><br><span class="line"> 因为这是进来一条统计一次 </span><br><span class="line">注意和Spark的区别： Spark里是一个批次的</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> stream.keyBy(_.domain).reduce((x,y) => {</span><br><span class="line"> Access(x.time, x.domain, (x.traffic+y.traffic+100))</span><br><span class="line"> }).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">5> Access(201912120010,ruozedata.com,4000)</span><br><span class="line">6> Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">6> Access(201912120010,dongqiudi.com,7100)</span><br><span class="line">5> Access(201912120010,ruozedata.com,6100)</span><br><span class="line">6> Access(201912120010,zhibo8.com,5000)</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">1.reduce 比sum灵活 keyby之后 是把相同的domain 放到一起了</span><br><span class="line">2.为什么zhibo8.com 结果没有＋100</span><br><span class="line"></span><br><span class="line">进来一条统计一次 为什么结果没有+100呢？</span><br><span class="line">因为 就1个呀 reduce 把相邻的两个做操作 1个操作啥</span><br></pre></td></tr></table></figure></div>
<p><strong>分流：</strong></p>
<p><img src="https://img-blog.csdnimg.cn/20191228131908308.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br></pre></td><td class="code"><pre><span class="line">这个功能 可能会用的到：</span><br><span class="line">这里使用split 后续 是使用 侧输出 因为split过时了 不过没有关系</span><br><span class="line"></span><br><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> // 5000 6000 7000</span><br><span class="line"> val splitStream = stream.keyBy("domain").sum("traffic").split(x => {</span><br><span class="line"> if (x.traffic > 6000) {</span><br><span class="line"> Seq("大客户")</span><br><span class="line"> } else {</span><br><span class="line"> Seq("一般客户")</span><br><span class="line"> }</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> splitStream.select("大客户").print("大客户")</span><br><span class="line"> splitStream.select("一般客户").print("一般客户")</span><br><span class="line"> </span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">一般客户:5> Access(201912120010,ruozedata.com,2000)</span><br><span class="line">一般客户:6> Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">一般客户:6> Access(201912120010,zhibo8.com,5000)</span><br><span class="line">一般客户:5> Access(201912120010,ruozedata.com,6000)</span><br><span class="line">大客户:6> Access(201912120010,dongqiudi.com,7000)</span><br><span class="line"></span><br><span class="line">splitStream.select("大客户", "一般客户").print("ALL")</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">一般客户:5> Access(201912120010,ruozedata.com,2000)</span><br><span class="line">一般客户:6> Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">ALL:5> Access(201912120010,ruozedata.com,2000)</span><br><span class="line">ALL:6> Access(201912120010,dongqiudi.com,1000)</span><br><span class="line">一般客户:6> Access(201912120010,zhibo8.com,5000)</span><br><span class="line">ALL:6> Access(201912120010,zhibo8.com,5000)</span><br><span class="line">一般客户:5> Access(201912120010,ruozedata.com,6000)</span><br><span class="line">大客户:6> Access(201912120010,dongqiudi.com,7000)</span><br><span class="line">ALL:5> Access(201912120010,ruozedata.com,6000)</span><br><span class="line">ALL:6> Access(201912120010,dongqiudi.com,7000)</span><br></pre></td></tr></table></figure></div>
<p><strong>合流</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br></pre></td><td class="code"><pre><span class="line">两个流是可以合的 ：</span><br><span class="line">union</span><br><span class="line">connect</span><br><span class="line"></span><br><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"> </span><br><span class="line"></span><br><span class="line"> val stream1 = env.addSource(new AccessSource)</span><br><span class="line"> val stream2 = env.addSource(new AccessSource)</span><br><span class="line"></span><br><span class="line">//1.数据类型是一样的 </span><br><span class="line"> stream1.union(stream2).map(x=>{</span><br><span class="line"> println("接收到的数据:" + x)</span><br><span class="line"> x</span><br><span class="line"> }).print()</span><br><span class="line"> </span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：不重要哈 结果不重要 </span><br><span class="line">接收到的数据:Access(1577510963163,dongqiudi.com,656)</span><br><span class="line">接收到的数据:Access(1577510963163,dongqiudi.com,523)</span><br><span class="line">接收到的数据:Access(1577510963163,dongqiudi.com,340)</span><br><span class="line">接收到的数据:Access(1577510963163,ruozedata.com,450)</span><br><span class="line">接收到的数据:Access(1577510963163,ruozedata.com,790)</span><br><span class="line">2> Access(1577510963163,ruozedata.com,450)</span><br><span class="line">接收到的数据:Access(1577510963163,dongqiudi.com,211)</span><br><span class="line">接收到的数据:Access(1577510963163,ruozedata.com,79)</span><br><span class="line">接收到的数据:Access(1577510963163,dongqiudi.com,673)</span><br><span class="line">4> Access(1577510963163,ruozedata.com,79)</span><br><span class="line">6> Access(1577510963163,dongqiudi.com,211)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line"> /**</span><br><span class="line"> * Creates a new DataStream by merging DataStream outputs of</span><br><span class="line"> * the same type with each other. The DataStreams merged using this operator</span><br><span class="line"> * will be transformed simultaneously.</span><br><span class="line"> *</span><br><span class="line"> */</span><br><span class="line"> def union(dataStreams: DataStream[T]*): DataStream[T] =</span><br><span class="line"> asScalaStream(stream.union(dataStreams.map(_.javaStream): _*))</span><br><span class="line">union: he same type with each other. 要求数据类型是一样的 两个流</span><br><span class="line"></span><br><span class="line">但是生产上 要合并两个流 很少 两个流数据类型是一样的 </span><br><span class="line"></span><br><span class="line"> /**</span><br><span class="line"> * Creates a new ConnectedStreams by connecting</span><br><span class="line"> * DataStream outputs of different type with each other. The</span><br><span class="line"> * DataStreams connected using this operators can be used with CoFunctions.</span><br><span class="line"> */</span><br><span class="line"> def connect[T2](dataStream: DataStream[T2]): ConnectedStreams[T, T2] =</span><br><span class="line"> asScalaStream(stream.connect(dataStream.javaStream))</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">两个流 数据类型不一样的： 使用 connect</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">两个流 数据类型不一样的：</span><br><span class="line"></span><br><span class="line">def map[R: TypeInformation](fun1: IN1 => R, fun2: IN2 => R):</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">fun1: IN1 => R, fun2: IN2 => R</span><br><span class="line"></span><br><span class="line">fun1: IN1 => R 第一个流 做的操作</span><br><span class="line"> fun2: IN2 => R 第二个流 做的操作</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br></pre></td><td class="code"><pre><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log").map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> val stream1 = env.addSource(new AccessSource)</span><br><span class="line"> val stream2 = env.addSource(new AccessSource)</span><br><span class="line"> </span><br><span class="line"> val stream2New = stream2.map(x => ("J哥", x))</span><br><span class="line"> </span><br><span class="line"> stream1.connect(stream2New).map(x=>x,y=>y).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">1> Access(1577511438837,zhibo8.cc,823)</span><br><span class="line">8> Access(1577511438837,zhibo8.cc,379)</span><br><span class="line">5> Access(1577511438837,zhibo8.cc,176)</span><br><span class="line">5> Access(1577511438837,zhibo8.cc,414)</span><br><span class="line">1> (J哥,Access(1577511438837,zhibo8.cc,553))</span><br><span class="line">2> Access(1577511438837,ruozedata.com,775)</span><br><span class="line">1> (J哥,Access(1577511438837,ruozedata.com,561))</span><br><span class="line">6> Access(1577511438837,dongqiudi.com,657)</span><br><span class="line">6> Access(1577511438837,zhibo8.cc,131)</span><br><span class="line">6> (J哥,Access(1577511438837,dongqiudi.com,826))</span><br><span class="line">7> Access(1577511438837,ruozedata.com,153)</span><br><span class="line">7> (J哥,Access(1577511438837,ruozedata.com,98))</span><br><span class="line">2> (J哥,Access(1577511438837,ruozedata.com,588))</span><br><span class="line">2> (J哥,Access(1577511438837,ruozedata.com,351))</span><br><span class="line">4> Access(1577511438837,ruozedata.com,24)</span><br><span class="line">8> (J哥,Access(1577511438837,dongqiudi.com,812))</span><br><span class="line">4> (J哥,Access(1577511438837,dongqiudi.com,686))</span><br><span class="line">5> (J哥,Access(1577511438837,zhibo8.cc,825))</span><br><span class="line">3> Access(1577511438837,dongqiudi.com,333)</span><br><span class="line">3> (J哥,Access(1577511438837,zhibo8.cc,140))</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">connect: </span><br><span class="line"> Connects two data streams retaining their types </span><br><span class="line"> 数据结构可以不同</span><br><span class="line"> two data streams **</span><br><span class="line">union：</span><br><span class="line"> Union of two or more data streams</span><br><span class="line"> 数据结构要相同</span><br><span class="line"> two or more data streams</span><br></pre></td></tr></table></figure></div>
<p><a href="https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/stream/operators/#physical-partitioning" target="_blank" rel="noopener">Physical partitioning</a></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre></td><td class="code"><pre><span class="line">class DoubleHappyPartitioner extends Partitioner[String]{</span><br><span class="line"> </span><br><span class="line"> override def partition(key: String, numPartitions: Int): Int = {</span><br><span class="line"> println("partitions: " + numPartitions)</span><br><span class="line">// 注意 scala 里面 不用使用 equals 直接 == 即可</span><br><span class="line"> if(key == "ruozedata.com"){</span><br><span class="line"> 0</span><br><span class="line"> } else if(key == "dongqiudi.com"){</span><br><span class="line"> 1</span><br><span class="line"> } else {</span><br><span class="line"> 2</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line">Partitioner[String]</span><br><span class="line"></span><br><span class="line">传进去的泛型 是 key的类型 </span><br><span class="line"></span><br><span class="line">分区的前提 是 kv</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">/**</span><br><span class="line"> * Partitions a tuple DataStream on the specified key fields using a custom partitioner.</span><br><span class="line"> * This method takes the key position to partition on, and a partitioner that accepts the key</span><br><span class="line"> * type.</span><br><span class="line"> *</span><br><span class="line"> * Note: This method works only on single field keys.</span><br><span class="line"> */</span><br><span class="line"> def partitionCustom[K: TypeInformation](partitioner: Partitioner[K], field: Int) : DataStream[T] =</span><br><span class="line"> asScalaStream(stream.partitionCustom(partitioner, field))</span><br><span class="line"></span><br><span class="line">field: Int 注意这个</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br></pre></td><td class="code"><pre><span class="line">object TranformationApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.setParallelism(3)</span><br><span class="line"></span><br><span class="line"> env.addSource(new AccessSource)</span><br><span class="line"> .map(x=>(x.domain, x))</span><br><span class="line"> .partitionCustom(new DoubleHappyPartitioner, 0)</span><br><span class="line"> .map(x => {</span><br><span class="line"> println("current thread id is: " + Thread.currentThread().getId + " , value is: " + x)</span><br><span class="line"> x._2</span><br><span class="line"> }).print()</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">partitions: 3</span><br><span class="line">current thread id is: 69 , value is: (zhibo8.cc,Access(1577512344598,zhibo8.cc,827))</span><br><span class="line">current thread id is: 68 , value is: (ruozedata.com,Access(1577512344598,ruozedata.com,143))</span><br><span class="line">1> Access(1577512344598,ruozedata.com,143)</span><br><span class="line">3> Access(1577512344598,zhibo8.cc,827)</span><br><span class="line">current thread id is: 68 , value is: (ruozedata.com,Access(1577512344598,ruozedata.com,672))</span><br><span class="line">1> Access(1577512344598,ruozedata.com,672)</span><br><span class="line">current thread id is: 70 , value is: (dongqiudi.com,Access(1577512344598,dongqiudi.com,450))</span><br><span class="line">2> Access(1577512344598,dongqiudi.com,450)</span><br><span class="line">current thread id is: 68 , value is: (ruozedata.com,Access(1577512344598,ruozedata.com,689))</span><br><span class="line">current thread id is: 69 , value is: (zhibo8.cc,Access(1577512344598,zhibo8.cc,608))</span><br><span class="line">current thread id is: 70 , value is: (dongqiudi.com,Access(1577512344598,dongqiudi.com,26))</span><br><span class="line">3> Access(1577512344598,zhibo8.cc,608)</span><br><span class="line">2> Access(1577512344598,dongqiudi.com,26)</span><br><span class="line">1> Access(1577512344598,ruozedata.com,689)</span><br><span class="line">current thread id is: 69 , value is: (zhibo8.cc,Access(1577512344598,zhibo8.cc,430))</span><br><span class="line">current thread id is: 70 , value is: (dongqiudi.com,Access(1577512344598,dongqiudi.com,514))</span><br><span class="line">3> Access(1577512344598,zhibo8.cc,430)</span><br><span class="line">2> Access(1577512344598,dongqiudi.com,514)</span><br><span class="line">current thread id is: 69 , value is: (zhibo8.cc,Access(1577512344598,zhibo8.cc,345))</span><br><span class="line">3> Access(1577512344598,zhibo8.cc,345)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">注意：线程</span><br><span class="line">1-> ruozedata.com</span><br><span class="line">2->dongqiudi.com</span><br><span class="line">3>zhibo8.cc </span><br><span class="line">结果是没有问题的</span><br><span class="line"></span><br><span class="line">这就是分区器的简单使用</span><br><span class="line">让每个线程处理不同的数据</span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-Flink01-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2020/01/05/Flink01-double-happy/">Flink01-double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2020/01/05/Flink01-double-happy/" class="article-date">
<time datetime="2020-01-05T15:43:46.000Z" itemprop="datePublished">2020-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="介绍"><a href="#介绍" class="headerlink" title="介绍"></a>介绍</h2><p><img src="https://img-blog.csdnimg.cn/20191224213557490.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line">Stateful Computations over Data Streams</span><br><span class="line"> 离线处理/批处理</span><br><span class="line"> state watermark</span><br><span class="line"></span><br><span class="line"> 事件驱动：</span><br><span class="line"> Flink ：来一条处理一条</span><br><span class="line"> Spark：是微批次的 eg：3s 处理 3s的这些数据</span><br><span class="line"> 流批一体：</span><br><span class="line"></span><br><span class="line"> Exactly-once state consistency</span><br><span class="line"> Event-time processing ***</span><br><span class="line"> Sophisticated late data handling 延迟数据的处理（延迟数据是避免不了的）</span><br><span class="line"> （我现在知道的 离线处理 Spark是可以解决延迟数据的问题）</span><br><span class="line"></span><br><span class="line">mvn archetype:generate \</span><br><span class="line">-DarchetypeGroupId=org.apache.flink \</span><br><span class="line">-DarchetypeArtifactId=flink-quickstart-scala \</span><br><span class="line">-DarchetypeVersion=1.9.0</span><br><span class="line"></span><br><span class="line"></span><br><span class="line"></span><br><span class="line">set up the batch execution environment</span><br><span class="line"> ExecutionEnvironment.getExecutionEnvironment</span><br><span class="line">Start with getting some data from the environment</span><br><span class="line">then, transform the resulting DataSet[String] using operations</span><br><span class="line">execute program</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">(x,...)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">RichXXXFunction</span><br><span class="line">生命周期函数</span><br><span class="line"> open 初始化方法</span><br><span class="line"> close 资源释放</span><br><span class="line"> getRuntimeContext 拿到整个作业运行时上下文</span><br></pre></td></tr></table></figure></div>
<p><strong>Run Applications at any Scale</strong><br>Flink is designed to run stateful streaming applications at any scale. Applications are parallelized into possibly thousands of tasks that are distributed and concurrently executed in a cluster. Therefore, an application can leverage virtually unlimited amounts of CPUs, main memory, disk and network IO. Moreover, Flink easily maintains very large application state. Its asynchronous and incremental checkpointing algorithm ensures minimal impact on processing latencies while guaranteeing exactly-once state consistency.</p>
<p>Users reported impressive scalability numbers for Flink applications running in their production environments, such as</p>
<p><strong>applications processing multiple trillions of events per day,</strong><br><strong>applications maintaining multiple terabytes of state, and<br>applications running on thousands of cores</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"> maintaining multiple terabytes of state：</span><br><span class="line"> 注意：</span><br><span class="line"> state维护在hdfs 有什么好处和坏处呢？</span><br><span class="line"></span><br><span class="line">Flink做的好的地方：</span><br><span class="line"> 1.处理时间 eventtime等 </span><br><span class="line"> 2.水印</span><br><span class="line"> 3.容错机制</span><br><span class="line"> 4.状态</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br><span class="line">99</span><br><span class="line">100</span><br><span class="line">101</span><br><span class="line">102</span><br><span class="line">103</span><br><span class="line">104</span><br><span class="line">105</span><br><span class="line">106</span><br><span class="line">107</span><br><span class="line">108</span><br><span class="line">109</span><br><span class="line">110</span><br><span class="line">111</span><br><span class="line">112</span><br></pre></td><td class="code"><pre><span class="line">pom.xml:</span><br><span class="line"></span><br><span class="line"><project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"</span><br><span class="line"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"></span><br><span class="line"> <modelVersion>4.0.0</modelVersion></span><br><span class="line"> <groupId>com.yt</groupId></span><br><span class="line"> <artifactId>Flink</artifactId></span><br><span class="line"> <version>1.0</version></span><br><span class="line"> <inceptionYear>2008</inceptionYear></span><br><span class="line"> <properties></span><br><span class="line"> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding></span><br><span class="line"> <flink.version>1.9.0</flink.version></span><br><span class="line"> <scala.binary.version>2.11</scala.binary.version></span><br><span class="line"> <scala.version>2.11.12</scala.version></span><br><span class="line"> <hadoop.version>2.7.2</hadoop.version></span><br><span class="line"> </properties></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> <repositories></span><br><span class="line"> <repository></span><br><span class="line"> <id>cloudera</id></span><br><span class="line"> <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url></span><br><span class="line"> </repository></span><br><span class="line"> </repositories></span><br><span class="line"></span><br><span class="line"></span><br><span class="line"> <dependencies></span><br><span class="line"> <dependency></span><br><span class="line"> <groupId>org.slf4j</groupId></span><br><span class="line"> <artifactId>slf4j-log4j12</artifactId></span><br><span class="line"> <version>1.7.7</version></span><br><span class="line"> <scope>runtime</scope></span><br><span class="line"> </dependency></span><br><span class="line"></span><br><span class="line"> <dependency></span><br><span class="line"> <groupId>log4j</groupId></span><br><span class="line"> <artifactId>log4j</artifactId></span><br><span class="line"> <version>1.2.17</version></span><br><span class="line"> <scope>runtime</scope></span><br><span class="line"> </dependency></span><br><span class="line"> <dependency></span><br><span class="line"> <groupId>org.apache.hadoop</groupId></span><br><span class="line"> <artifactId>hadoop-client</artifactId></span><br><span class="line"> <version>${hadoop.version}</version></span><br><span class="line"> </dependency></span><br><span class="line"> <!--Flink依赖--></span><br><span class="line"> <dependency></span><br><span class="line"> <groupId>org.apache.flink</groupId></span><br><span class="line"> <artifactId>flink-scala_${scala.binary.version}</artifactId></span><br><span class="line"> <version>${flink.version}</version></span><br><span class="line"> </dependency></span><br><span class="line"> <dependency></span><br><span class="line"> <groupId>org.apache.flink</groupId></span><br><span class="line"> <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId></span><br><span class="line"> <version>${flink.version}</version></span><br><span class="line"> </dependency></span><br><span class="line"></span><br><span class="line"> </dependencies></span><br><span class="line"></span><br><span class="line"> <build></span><br><span class="line"> <sourceDirectory>src/main/scala</sourceDirectory></span><br><span class="line"> <testSourceDirectory>src/test/scala</testSourceDirectory></span><br><span class="line"> <plugins></span><br><span class="line"> <plugin></span><br><span class="line"> <groupId>org.scala-tools</groupId></span><br><span class="line"> <artifactId>maven-scala-plugin</artifactId></span><br><span class="line"> <executions></span><br><span class="line"> <execution></span><br><span class="line"> <goals></span><br><span class="line"> <goal>compile</goal></span><br><span class="line"> <goal>testCompile</goal></span><br><span class="line"> </goals></span><br><span class="line"> </execution></span><br><span class="line"> </executions></span><br><span class="line"> <configuration></span><br><span class="line"> <scalaVersion>${scala.version}</scalaVersion></span><br><span class="line"> <args></span><br><span class="line"> <arg>-target:jvm-1.5</arg></span><br><span class="line"> </args></span><br><span class="line"> </configuration></span><br><span class="line"> </plugin></span><br><span class="line"> <plugin></span><br><span class="line"> <groupId>org.apache.maven.plugins</groupId></span><br><span class="line"> <artifactId>maven-eclipse-plugin</artifactId></span><br><span class="line"> <configuration></span><br><span class="line"> <downloadSources>true</downloadSources></span><br><span class="line"> <buildcommands></span><br><span class="line"> <buildcommand>ch.epfl.lamp.sdt.core.scalabuilder</buildcommand></span><br><span class="line"> </buildcommands></span><br><span class="line"> <additionalProjectnatures></span><br><span class="line"> <projectnature>ch.epfl.lamp.sdt.core.scalanature</projectnature></span><br><span class="line"> </additionalProjectnatures></span><br><span class="line"> <classpathContainers></span><br><span class="line"> <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer></span><br><span class="line"> <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINER</classpathContainer></span><br><span class="line"> </classpathContainers></span><br><span class="line"> </configuration></span><br><span class="line"> </plugin></span><br><span class="line"> </plugins></span><br><span class="line"> </build></span><br><span class="line"> <reporting></span><br><span class="line"> <plugins></span><br><span class="line"> <plugin></span><br><span class="line"> <groupId>org.scala-tools</groupId></span><br><span class="line"> <artifactId>maven-scala-plugin</artifactId></span><br><span class="line"> <configuration></span><br><span class="line"> <scalaVersion>${scala.version}</scalaVersion></span><br><span class="line"> </configuration></span><br><span class="line"> </plugin></span><br><span class="line"> </plugins></span><br><span class="line"> </reporting></span><br><span class="line"></project></span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre></td><td class="code"><pre><span class="line">批处理：</span><br><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import org.apache.flink.api.scala.{DataSet, ExecutionEnvironment}</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line">object BatchJob {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> // 获取批处理上下文 <== SparkContext</span><br><span class="line"> val env = ExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> // 读取数据</span><br><span class="line"> val text: DataSet[String] = env.readTextFile("C:\\IdeaProjects\\flink\\data\\data.txt")</span><br><span class="line"></span><br><span class="line"> val value: DataSet[(String, Int)] = text.flatMap(_.toLowerCase.split(","))</span><br><span class="line"> .filter(_.nonEmpty)</span><br><span class="line"> .map((_, 1))</span><br><span class="line"></span><br><span class="line"> // transformation</span><br><span class="line"> val result = value.groupBy(0) // 0表示单词</span><br><span class="line"> .sum(1)</span><br><span class="line"></span><br><span class="line"> // sink output</span><br><span class="line"> result.print()</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">(double_happy,1)</span><br><span class="line">(kite,1)</span><br><span class="line">(25,2)</span><br><span class="line">(kairis,1)</span><br><span class="line">(32,1)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 ~]$ nc -lk 7777</span><br><span class="line">doublehappy,1</span><br><span class="line">doublehappy,2</span><br><span class="line">doublehappy,3</span><br><span class="line">-------</span><br><span class="line"></span><br><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object StreamingJob {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> // set up the streaming execution environment</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> </span><br><span class="line"> // 接收数据</span><br><span class="line"> val text: DataStream[String] = env.socketTextStream("hadoop101", 7777)</span><br><span class="line"></span><br><span class="line"> // transformation</span><br><span class="line"> val result = text.flatMap(_.toLowerCase.split(","))</span><br><span class="line"> .filter(_.nonEmpty)</span><br><span class="line"> .map((_, 1))</span><br><span class="line"> .keyBy(0).sum(1).print()</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果是：</span><br><span class="line">2> (2,1)</span><br><span class="line">4> (doublehappy,1)</span><br><span class="line">4> (1,1)</span><br><span class="line">4> (doublehappy,2)</span><br><span class="line">4> (doublehappy,3)</span><br><span class="line">3> (3,1)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre></td><td class="code"><pre><span class="line">[double_happy@hadoop101 ~]$ nc -lk 7777</span><br><span class="line">double,double,double</span><br><span class="line">happy,happy</span><br><span class="line">------------------------</span><br><span class="line"></span><br><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object StreamingJob {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"></span><br><span class="line"> // set up the streaming execution environment</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> // 接收数据</span><br><span class="line"> val text: DataStream[String] = env.socketTextStream("hadoop101", 7777)</span><br><span class="line"></span><br><span class="line"> // transformation</span><br><span class="line"> val result = text.flatMap(_.toLowerCase.split(","))</span><br><span class="line"> .filter(_.nonEmpty)</span><br><span class="line"> .map((_, 1))</span><br><span class="line"> .keyBy(0).sum(1).print("double_happy")</span><br><span class="line"> .setParallelism(2)</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">double_happy:1> (double,2)</span><br><span class="line">double_happy:2> (double,1)</span><br><span class="line">double_happy:2> (double,3)</span><br><span class="line">double_happy:1> (happy,1)</span><br><span class="line">double_happy:2> (happy,2)</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">并行度为1：结果</span><br><span class="line">double_happy> (double,1)</span><br><span class="line">double_happy> (double,2)</span><br><span class="line">double_happy> (double,3)</span><br><span class="line">double_happy> (happy,1)</span><br><span class="line">double_happy> (happy,2)</span><br></pre></td></tr></table></figure></div>
<p>使用字段表达式</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br></pre></td><td class="code"><pre><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object StreamingJob {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"></span><br><span class="line"> // set up the streaming execution environment</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"></span><br><span class="line"> // 接收数据</span><br><span class="line"> val text: DataStream[String] = env.socketTextStream("hadoop101", 7777)</span><br><span class="line"></span><br><span class="line"> // transformation</span><br><span class="line">// val result = text.flatMap(_.toLowerCase.split(","))</span><br><span class="line">// .filter(_.nonEmpty)</span><br><span class="line">// .map((_, 1))</span><br><span class="line">// .keyBy(0).sum(1).print("double_happy")</span><br><span class="line">// .setParallelism(1)</span><br><span class="line"></span><br><span class="line"> val result = text.flatMap(_.toLowerCase.split(","))</span><br><span class="line"> .filter(_.nonEmpty)</span><br><span class="line"> .map(x => WC(x, 1))</span><br><span class="line"> .keyBy(_.word).sum("count").print("double_happy")</span><br><span class="line"> .setParallelism(1)</span><br><span class="line">//</span><br><span class="line"> // execute program</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"></span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">case class WC(word: String, count: Int)</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191224232202396.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><img src="https://img-blog.csdnimg.cn/20191224232123302.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<p><img src="https://img-blog.csdnimg.cn/20191224232252212.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<h2 id="依赖"><a href="#依赖" class="headerlink" title="依赖"></a>依赖</h2><p><img src="https://img-blog.csdnimg.cn/20191224225316454.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">At the bare minimum, the application depends on the Flink APIs. Many applications depend in addition on certain connector libraries (like Kafka, Cassandra, etc.)</span><br><span class="line"></span><br><span class="line">注意：</span><br><span class="line"> connector libraries</span><br></pre></td></tr></table></figure></div>
<p><strong>Specifying Transformation Functions</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">package com.sx.bean</span><br><span class="line"></span><br><span class="line">object Domain {</span><br><span class="line"> case class Access(time:Long, domain:String, traffic:Long)</span><br><span class="line"></span><br><span class="line">}</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br></pre></td><td class="code"><pre><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import com.sx.bean.Domain.Access</span><br><span class="line">import org.apache.flink.api.common.functions.{FilterFunction, RichMapFunction, RuntimeContext}</span><br><span class="line">import org.apache.flink.configuration.Configuration</span><br><span class="line">import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object SpecifyingTransformationFunctionsApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.setParallelism(1)</span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log")</span><br><span class="line"> val accessStream = stream.map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"></span><br><span class="line"> accessStream.print()</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">Access(201912120010,ruozedata.com,2000)</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">Access(201912120010,zhibo8.com,5000)</span><br><span class="line">Access(201912120010,ruozedata.com,4000)</span><br><span class="line">Access(201912120010,dongqiudi.com,1000)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">需求1</span><br><span class="line"> 过滤traffic > 4000</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">方式1：Lambda Functions</span><br><span class="line">object SpecifyingTransformationFunctionsApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.setParallelism(1)</span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log")</span><br><span class="line"> val accessStream = stream.map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"> // 过滤traffic > 4000</span><br><span class="line"> accessStream.filter(_.traffic > 4000).print()</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">Access(201912120010,zhibo8.com,5000)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br></pre></td><td class="code"><pre><span class="line">方式二：</span><br><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import com.sx.bean.Domain.Access</span><br><span class="line">import org.apache.flink.api.common.functions.{FilterFunction, RichMapFunction, RuntimeContext}</span><br><span class="line">import org.apache.flink.configuration.Configuration</span><br><span class="line">import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object SpecifyingTransformationFunctionsApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.setParallelism(1)</span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log")</span><br><span class="line"> val accessStream = stream.map(x => {</span><br><span class="line"> val splits = x.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> })</span><br><span class="line"> accessStream.filter(new DoubleHappyFilter02(5000)).print()</span><br><span class="line"> </span><br><span class="line"> // accessStream.filter(new FilterFunction[Access] {</span><br><span class="line"> // override def filter(value: Access): Boolean = value.traffic>4000</span><br><span class="line"> // }).print() //匿名内部类写法 别这么写 太难看了</span><br><span class="line"></span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">// XXXFunction RichXXXFunction</span><br><span class="line">class DoubleHappyFilter extends FilterFunction[Access] {</span><br><span class="line"> override def filter(value: Access): Boolean = value.traffic > 4000</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">class DoubleHappyFilter02(traffic: Long) extends FilterFunction[Access] {</span><br><span class="line"> override def filter(value: Access): Boolean = value.traffic > traffic</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">结果：</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20191224234229488.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"><br><strong>这里面的生命周期函数</strong></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">RichXXXFunction</span><br><span class="line">生命周期函数</span><br><span class="line"> open 初始化方法</span><br><span class="line"> close 资源释放</span><br><span class="line"> getRuntimeContext 拿到整个作业运行时上下文</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br></pre></td><td class="code"><pre><span class="line">方式3：Rich functions</span><br><span class="line"></span><br><span class="line">package com.sx.flink01</span><br><span class="line"></span><br><span class="line">import com.sx.bean.Domain.Access</span><br><span class="line">import org.apache.flink.api.common.functions.{FilterFunction, RichMapFunction, RuntimeContext}</span><br><span class="line">import org.apache.flink.configuration.Configuration</span><br><span class="line">import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment</span><br><span class="line">import org.apache.flink.api.scala._</span><br><span class="line"></span><br><span class="line">object SpecifyingTransformationFunctionsApp {</span><br><span class="line"></span><br><span class="line"> def main(args: Array[String]): Unit = {</span><br><span class="line"> val env = StreamExecutionEnvironment.getExecutionEnvironment</span><br><span class="line"> env.setParallelism(2)</span><br><span class="line"> </span><br><span class="line"> val stream = env.readTextFile("C:\\IdeaProjects\\flink\\data\\access.log")</span><br><span class="line"> val accessStream = stream.map(new DoubleHappyMap)</span><br><span class="line"></span><br><span class="line"> // 过滤traffic > 4000</span><br><span class="line"> accessStream.filter(_.traffic > 4000).print()</span><br><span class="line"> env.execute(this.getClass.getSimpleName)</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">// XXXFunction RichXXXFunction</span><br><span class="line">class DoubleHappyFilter extends FilterFunction[Access] {</span><br><span class="line"> override def filter(value: Access): Boolean = value.traffic > 4000</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">class DoubleHappyFilter02(traffic: Long) extends FilterFunction[Access] {</span><br><span class="line"> override def filter(value: Access): Boolean = value.traffic > traffic</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line">class DoubleHappyMap extends RichMapFunction[String, Access] {</span><br><span class="line"> override def map(value: String): Access = {</span><br><span class="line"> val splits = value.split(",")</span><br><span class="line"> Access(splits(0).toLong, splits(1), splits(2).toLong)</span><br><span class="line"> }</span><br><span class="line"></span><br><span class="line"> override def open(parameters: Configuration): Unit = {</span><br><span class="line"> super.open(parameters)</span><br><span class="line"> println("~~~~~~~~~~~open~~~~~~~")</span><br><span class="line"> }</span><br><span class="line"> override def close(): Unit = {</span><br><span class="line"> super.close()</span><br><span class="line"> }</span><br><span class="line"> override def getRuntimeContext: RuntimeContext = {</span><br><span class="line"> super.getRuntimeContext</span><br><span class="line"> }</span><br><span class="line">}</span><br><span class="line"></span><br><span class="line"></span><br><span class="line">结果： 并行度为2的时候 </span><br><span class="line">~~~~~~~~~~~open~~~~~~~</span><br><span class="line">~~~~~~~~~~~open~~~~~~~</span><br><span class="line">1> Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">1> Access(201912120010,zhibo8.com,5000)</span><br><span class="line"></span><br><span class="line">这块要注意 ：MySQL获取connection 放在open方法里 </span><br><span class="line">还要注意 不同的并行读 调用了几次 </span><br><span class="line"></span><br><span class="line">结果：并行度为1的时候</span><br><span class="line">~~~~~~~~~~~open~~~~~~~</span><br><span class="line">Access(201912120010,dongqiudi.com,6000)</span><br><span class="line">Access(201912120010,zhibo8.com,5000)</span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-tmp" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2019/11/20/tmp/">tmp</a>
</h1>
</header>
<div class="article-meta">
<a href="/2019/11/20/tmp/" class="article-date">
<time datetime="2019-11-19T16:45:42.000Z" itemprop="datePublished">2019-11-20</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
</div>
</div>
</article>
<article id="post-Azkaban调度-double-happy" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2019/01/05/Azkaban%E8%B0%83%E5%BA%A6-double-happy/">Azkaban调度--double_happy</a>
</h1>
</header>
<div class="article-meta">
<a href="/2019/01/05/Azkaban%E8%B0%83%E5%BA%A6-double-happy/" class="article-date">
<time datetime="2019-01-05T11:56:15.000Z" itemprop="datePublished">2019-01-05</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="安装Azkaban的坑"><a href="#安装Azkaban的坑" class="headerlink" title="安装Azkaban的坑"></a>安装Azkaban的坑</h2><p>当你使用单机模式运行azkaban时候：</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">2019/09/23 22:06:53.626 +0800 ERROR [AzkabanWebServer] [Azkaban] Failed to start single server. Shutting down.</span><br><span class="line">java.io.IOException: Cannot find 'database.properties' file in /home/double_happy/app/azkaban-solo-server/bin/sql/database.properties</span><br><span class="line"> at azkaban.database.AzkabanDatabaseSetup.loadDBProps(AzkabanDatabaseSetup.java:178)</span><br><span class="line"> at azkaban.database.AzkabanDatabaseSetup.loadTableInfo(AzkabanDatabaseSetup.java:102)</span><br><span class="line"> at azkaban.database.AzkabanDatabaseUpdater.runDatabaseUpdater(AzkabanDatabaseUpdater.java:82)</span><br><span class="line"> at azkaban.soloserver.AzkabanSingleServer.start(AzkabanSingleServer.java:93)</span><br><span class="line"> at azkaban.soloserver.AzkabanSingleServer.main(AzkabanSingleServer.java:58)</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">解决方式：</span><br><span class="line"></span><br><span class="line">必须使用bin/azkaban-solo-start.sh</span><br></pre></td></tr></table></figure></div>
<h2 id="调度解决多个作业之间的依赖关系"><a href="#调度解决多个作业之间的依赖关系" class="headerlink" title="调度解决多个作业之间的依赖关系"></a>调度解决多个作业之间的依赖关系</h2><div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">nodes:</span><br><span class="line"> - name: platform_stat</span><br><span class="line"> type: command</span><br><span class="line"> dependsOn:</span><br><span class="line"> - etl</span><br><span class="line"> config:</span><br><span class="line"> command: sh /home/double_happy/ruozedata/project/hadoop-project/shell/platform_stat.sh 20190921</span><br><span class="line"></span><br><span class="line"> - name: etl</span><br><span class="line"> type: command</span><br><span class="line"> config:</span><br><span class="line"> command: sh /home/double_happy/ruozedata/project/hadoop-project/shell/etl.sh 20190921</span><br></pre></td></tr></table></figure></div>
<p><img src="https://img-blog.csdnimg.cn/20190923234758109.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9kb3VibGVoYXBweS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70" alt="在这里插入图片描述"></p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">但是又一个问题呀：</span><br><span class="line"> shell 里的参数我可以不可以在 azkaban上面通过 parameters 传进来呢？</span><br><span class="line"> 待续。。。。。。。</span><br></pre></td></tr></table></figure></div>
</div>
</div>
</article>
<article id="post-Zookeeper基本使用与监控-Curator" class="article global-container article-type-post" itemscope itemprop="blogPost">
<header class="article-header">
<h1 itemprop="name">
<a class="article-title" href="/2019/01/04/Zookeeper%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8%E4%B8%8E%E7%9B%91%E6%8E%A7-Curator/">Zookeeper基本使用与监控(Curator)</a>
</h1>
</header>
<div class="article-meta">
<a href="/2019/01/04/Zookeeper%E5%9F%BA%E6%9C%AC%E4%BD%BF%E7%94%A8%E4%B8%8E%E7%9B%91%E6%8E%A7-Curator/" class="article-date">
<time datetime="2019-01-04T11:46:26.000Z" itemprop="datePublished">2019-01-04</time>
</a>
</div>
<div class="article-inner">
<div class="article-content article-content-cloud" itemprop="articleBody">
<h2 id="1-Zookeeper基本使用"><a href="#1-Zookeeper基本使用" class="headerlink" title="1.Zookeeper基本使用"></a>1.Zookeeper基本使用</h2><p>(0)基本了解</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">ZK核心术语</span><br><span class="line"> ZK的数据模型： 树形结构</span><br><span class="line"> /home/hadoop/app/ruozedata.txt</span><br><span class="line"> ZK每个节点znode，有一个唯一的路径标识</span><br><span class="line"> znode</span><br><span class="line"> 临时的 ephemeral 临时znode下面不能有子节点</span><br><span class="line"> 普通的</span><br><span class="line"> sequential</span><br><span class="line"> 永久的 persistent</span><br><span class="line"> 普通的</span><br><span class="line"> sequential 顺序编号目录节点</span><br><span class="line"> 每个znode节点有各自的版本号</span><br><span class="line"> 每个节点数据发生了变化，该节点的版本会加</span><br><span class="line"> ZK节点存储的数据量不宜过大，几K *****</span><br><span class="line"> znode可以设置访问权限</span><br><span class="line"> znode可以设置watcher：当节点数据发生变化时，可以通过监视器获取</span><br></pre></td></tr></table></figure></div>
<p>(1)客户端命令行操作</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line"></span><br><span class="line"> 命令基本语法 功能描述</span><br><span class="line">help 显示所有操作命令</span><br><span class="line">ls path [watch] 使用 ls 命令来查看当前znode中所包含的内容</span><br><span class="line">ls2 path [watch] 查看当前节点数据并能看到更新次数等数据</span><br><span class="line">create 普通创建</span><br><span class="line"> -s 含有序列</span><br><span class="line"> -e 临时（重启或者超时消失）</span><br><span class="line">get path [watch] 获得节点的值</span><br><span class="line">set 设置节点的具体值</span><br><span class="line">stat 查看节点状态</span><br><span class="line">delete 删除节点</span><br><span class="line">rmr 递归删除节点</span><br></pre></td></tr></table></figure></div>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre></td><td class="code"><pre><span class="line">上面带[watch] 的是可以监控的 但是watcher zookeeper自身的api和命令都是一次性的 </span><br><span class="line">也就是用过了不能再用了（你懂的 ）</span><br><span class="line">解决办法 可以使用 curator 作为zkClient 来监控zookeeper的变化信息****</span><br><span class="line"></span><br><span class="line">eg：一些监控的种类</span><br><span class="line">watcher</span><br><span class="line"> zk中的watcher是一次性(*****)的</span><br><span class="line"> NodeCreated</span><br><span class="line"> NodeDataChanged</span><br><span class="line"> NodeDeleted</span><br><span class="line"> NodeChildrenChanged</span><br><span class="line"> </span><br><span class="line">四字命令 了解一下</span><br></pre></td></tr></table></figure></div>
<p>(2)CuratorApi<br>Curator是Netflix公司开源的一套zookeeper客户端框架，解决了很多Zookeeper客户端非常底层的细节开发工作，包括连接重连、反复注册Watcher和NodeExistsException异常等等。</p>
<p>版本选择：</p>
<div class="highlight-box"autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false" contenteditable="true"data-rel="PLAIN"><figure class="iseeu highlight /plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">这里我们的zookeeper的版本是3.4.x，所以api开发的时候 maven不同版本会用不同</span><br><span class="line">https://curator.apache.org/zk-compatibility.html curator官网</span><br></pre></td></tr></table></figure></div>
<p>curator api 官网 ：<a href="https://curator.apache.org/curator-framework/index.html" target="_blank" rel="noopener">https://curator.apache.org/curator-framework/index.html</a></p>
<p>可以去官网上去学习，可以结合zookeeper api 来学习 curator api （突然手里来活了 写不了了 ）<br>这里可以借鉴：<br> <a href="http://www.throwable.club/2018/12/16/zookeeper-curator-usage/#%E5%89%8D%E6%8F%90" target="_blank" rel="noopener">http://www.throwable.club/2018/12/16/zookeeper-curator-usage/#%E5%89%8D%E6%8F%90</a><br>英文不好的小伙伴 ，这篇文章很好的介绍Curator 使用 。</p>
<p>还有我的小组长的博客：<br><a href="https://liverrrr.fun/archives/quickstart_zookeeper" target="_blank" rel="noopener">https://liverrrr.fun/archives/quickstart_zookeeper</a><br>可以很好的学习 。</p>
</div>
</div>
</article>
<nav id="page-nav">
<span class="page-number current">1</span><a class="page-number" href="/page/2/">2</a><a class="page-number" href="/page/3/">3</a><span class="space">…</span><a class="page-number" href="/page/6/">6</a><a class="extend next" rel="next" href="/page/2/">Next &raquo;</a>
</nav>
</section>
<aside id="sidebar">
<div class="widget-box">
<div class="avatar-box">
<img class="avatar" src="/images/default-avatar.jpg" title="图片来自网络"></img>
<h3 class="avatar-name">
DoubleHappy
</h3>
<p class="avatar-slogan">
特别耐撕的大数据，资深的打酱油攻城狮。
</p>
</div>
</div>
<div class="widget-box">
<h3 class="widget-title">Archives</h3>
<div class="widget">
<ul class="archive-list"><li class="archive-list-item"><a class="archive-list-link" href="/archives/2020/04/">April 2020</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2020/01/">January 2020</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2019/11/">November 2019</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2019/01/">January 2019</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/04/">April 2018</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/03/">March 2018</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/02/">February 2018</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2018/01/">January 2018</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2017/12/">December 2017</a></li><li class="archive-list-item"><a class="archive-list-link" href="/archives/2017/11/">November 2017</a></li></ul>
</div>
</div>
<div class="widget-box">
<h3 class="widget-title">Recent Posts</h3>
<div class="widget">
<ul>
<li>
<a href="/2020/04/27/docker-doublehappy/">docker--doublehappy</a>
</li>
<li>
<a href="/2020/04/27/k8s-Spark-doublehappy/">k8s-Spark-doublehappy</a>
</li>
<li>
<a href="/2020/01/05/Kudu-Impala%E6%95%85%E9%9A%9C%E6%A1%88%E4%BE%8B01-double-happy/">Kudu+Impala故障案例01--double_happy</a>
</li>
<li>
<a href="/2020/01/05/Flink04-double-happy/">Flink04--double_happy</a>
</li>
<li>
<a href="/2020/01/05/Flink03-double-happy/">Flink03--double_happy</a>
</li>
</ul>
</div>
</div>
<div class="widget-box">
<h3 class="widget-title">友链</h3>
<div class="widget">
<a style="display: block;" href="https://liverrrr.fun/archives" title target='_blank'
>一路眼瞎</a>
</div>
</div>
</aside>
</div>
<footer id="footer">
<div class="foot-box global-width">
© 2020 DoubleHappy
Powered by <a href="http://hexo.io/" target="_blank">Hexo</a>
| 主题 <a href="https://github.com/yiluyanxia/hexo-theme-antiquity" target="_blank" rel="noopener">antiquity</a>
<br>
<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
<span id="busuanzi_container_site_pv">不蒜子告之 阁下是第<span id="busuanzi_value_site_pv"></span>个访客</span>
</div>
</footer>
<script src="https://code.jquery.com/jquery-2.0.3.min.js"></script>
<script>
if (!window.jQuery) {
var script = document.createElement('script');
script.src = "/js/jquery-2.0.3.min.js";
document.body.write(script);
}
</script>
<script src="/js/script.js"></script>
</div>
<nav id="mobile-nav" class="mobile-nav-box">
<div class="mobile-nav-img mobile-nav-top"></div>
<a href="/archives" class="mobile-nav-link">首页</a>
<a href="/archives" class="mobile-nav-link">归档</a>
<div class="mobile-nav-img mobile-nav-bottom"></div>
</nav>
</div>
</body>
</html>