-
Notifications
You must be signed in to change notification settings - Fork 15
Expand file tree
/
Copy pathindex.tex
More file actions
4226 lines (3545 loc) · 246 KB
/
index.tex
File metadata and controls
4226 lines (3545 loc) · 246 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor}
%
\documentclass[
letterpaper,
DIV=11,
numbers=noendperiod]{scrreprt}
\usepackage{amsmath,amssymb}
\usepackage{iftex}
\ifPDFTeX
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
\usepackage{unicode-math}
\defaultfontfeatures{Scale=MatchLowercase}
\defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
\fi
\usepackage{lmodern}
\ifPDFTeX\else
% xetex/luatex font selection
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
\usepackage[]{microtype}
\UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
\IfFileExists{parskip.sty}{%
\usepackage{parskip}
}{% else
\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
\KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\setlength{\emergencystretch}{3em} % prevent overfull lines
\setcounter{secnumdepth}{5}
% Make \paragraph and \subparagraph free-standing
\makeatletter
\ifx\paragraph\undefined\else
\let\oldparagraph\paragraph
\renewcommand{\paragraph}{
\@ifstar
\xxxParagraphStar
\xxxParagraphNoStar
}
\newcommand{\xxxParagraphStar}[1]{\oldparagraph*{#1}\mbox{}}
\newcommand{\xxxParagraphNoStar}[1]{\oldparagraph{#1}\mbox{}}
\fi
\ifx\subparagraph\undefined\else
\let\oldsubparagraph\subparagraph
\renewcommand{\subparagraph}{
\@ifstar
\xxxSubParagraphStar
\xxxSubParagraphNoStar
}
\newcommand{\xxxSubParagraphStar}[1]{\oldsubparagraph*{#1}\mbox{}}
\newcommand{\xxxSubParagraphNoStar}[1]{\oldsubparagraph{#1}\mbox{}}
\fi
\makeatother
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{241,243,245}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.40,0.45,0.13}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\BuiltInTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\ExtensionTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.28,0.35,0.67}{#1}}
\newcommand{\ImportTok}[1]{\textcolor[rgb]{0.00,0.46,0.62}{#1}}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.68,0.00,0.00}{#1}}
\newcommand{\RegionMarkerTok}[1]{\textcolor[rgb]{0.00,0.23,0.31}{#1}}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.07,0.07,0.07}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.13,0.47,0.30}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.37,0.37,0.37}{\textit{#1}}}
\providecommand{\tightlist}{%
\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}\usepackage{longtable,booktabs,array}
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
% definitions for citeproc citations
\NewDocumentCommand\citeproctext{}{}
\NewDocumentCommand\citeproc{mm}{%
\begingroup\def\citeproctext{#2}\cite{#1}\endgroup}
\makeatletter
% allow citations to break across lines
\let\@cite@ofmt\@firstofone
% avoid brackets around text for \cite:
\def\@biblabel#1{}
\def\@cite#1#2{{#1\if@tempswa , #2\fi}}
\makeatother
\newlength{\cslhangindent}
\setlength{\cslhangindent}{1.5em}
\newlength{\csllabelwidth}
\setlength{\csllabelwidth}{3em}
\newenvironment{CSLReferences}[2] % #1 hanging-indent, #2 entry-spacing
{\begin{list}{}{%
\setlength{\itemindent}{0pt}
\setlength{\leftmargin}{0pt}
\setlength{\parsep}{0pt}
% turn on hanging indent if param 1 is 1
\ifodd #1
\setlength{\leftmargin}{\cslhangindent}
\setlength{\itemindent}{-1\cslhangindent}
\fi
% set entry spacing
\setlength{\itemsep}{#2\baselineskip}}}
{\end{list}}
\usepackage{calc}
\newcommand{\CSLBlock}[1]{\hfill\break\parbox[t]{\linewidth}{\strut\ignorespaces#1\strut}}
\newcommand{\CSLLeftMargin}[1]{\parbox[t]{\csllabelwidth}{\strut#1\strut}}
\newcommand{\CSLRightInline}[1]{\parbox[t]{\linewidth - \csllabelwidth}{\strut#1\strut}}
\newcommand{\CSLIndent}[1]{\hspace{\cslhangindent}#1}
\KOMAoption{captions}{tableheading}
\makeatletter
\@ifpackageloaded{bookmark}{}{\usepackage{bookmark}}
\makeatother
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\AtBeginDocument{%
\ifdefined\contentsname
\renewcommand*\contentsname{Table of contents}
\else
\newcommand\contentsname{Table of contents}
\fi
\ifdefined\listfigurename
\renewcommand*\listfigurename{List of Figures}
\else
\newcommand\listfigurename{List of Figures}
\fi
\ifdefined\listtablename
\renewcommand*\listtablename{List of Tables}
\else
\newcommand\listtablename{List of Tables}
\fi
\ifdefined\figurename
\renewcommand*\figurename{Figure}
\else
\newcommand\figurename{Figure}
\fi
\ifdefined\tablename
\renewcommand*\tablename{Table}
\else
\newcommand\tablename{Table}
\fi
}
\@ifpackageloaded{float}{}{\usepackage{float}}
\floatstyle{ruled}
\@ifundefined{c@chapter}{\newfloat{codelisting}{h}{lop}}{\newfloat{codelisting}{h}{lop}[chapter]}
\floatname{codelisting}{Listing}
\newcommand*\listoflistings{\listof{codelisting}{List of Listings}}
\makeatother
\makeatletter
\makeatother
\makeatletter
\@ifpackageloaded{caption}{}{\usepackage{caption}}
\@ifpackageloaded{subcaption}{}{\usepackage{subcaption}}
\makeatother
\ifLuaTeX
\usepackage{selnolig} % disable illegal ligatures
\fi
\usepackage{bookmark}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\urlstyle{same} % disable monospaced font for URLs
\hypersetup{
pdftitle={Applied Artificial Intelligence and Deep Learning Book},
pdfauthor={Tim Mayer; Biplov Bhandari; David Saah},
colorlinks=true,
linkcolor={blue},
filecolor={Maroon},
citecolor={Blue},
urlcolor={Blue},
pdfcreator={LaTeX via pandoc}}
\title{Applied Artificial Intelligence and Deep Learning Book}
\author{Tim Mayer \and Biplov Bhandari \and David Saah}
\date{2025-09-18}
\begin{document}
\maketitle
\renewcommand*\contentsname{Table of contents}
{
\hypersetup{linkcolor=}
\setcounter{tocdepth}{2}
\tableofcontents
}
\bookmarksetup{startatroot}
\chapter*{Introduction}\label{introduction}
\addcontentsline{toc}{chapter}{Introduction}
\markboth{Introduction}{Introduction}
The NASA EarthRISE program harnesses NASA Earth Action capabilities to
deliver trusted solutions for sustained benefits to society.
Collabroting with State and Local partners and striving to strengthens
the capaciies of cof parterns to use satellite data and geospatial
technology to address critical challenges. EarthRISE co-develops
innovative solutions through a network of partners to improve resilience
and sustainable resource management across scales. Additionally,
EarthRISE focuses on developing participate in innovative knowledge
products such as the SAR Handbook and the GEE book designed to support
capacity building in applying Remote Sensing and geospatial approaches
to address challenges.
The focus of the EarthRISE Applied Artificial Intelligence and Deep
Learning Book is to provide practitioners with a wide variety of applied
examples of Remote Sensing Artificial Intelligence and Deep Learning
approaches. With each chapter focusing on a specific problem set such as
object detection of downscaling using Deep Learning. Additionally,
throughout the books chapters various examples are provided spanning the
aforementioned NASA Earth Action thematic areas. Thereby providing a
wide variety of thematic applications to complement reader's domain
specific practical knowledge such as agronomy or forestry etc.
We suspect readers are coming to this virtual book with preexisting
geospatial expertise. However, limited Artificial Intelligence and Deep
Learning knowledge
Each chapter contains both the theoretical background as well as a
practical hand-on section facilitated through virtual notebooks.
Finally, this book spans a variety of platforms such as TensorFlow and
PyTorch to provide readers with a wide set of examples.
\part{Curriculum}
\chapter{Data Preparation}\label{data-preparation}
\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Print out the Python version used by this environment.}
\ImportTok{import}\NormalTok{ sys}
\BuiltInTok{print}\NormalTok{(}\SpecialStringTok{f\textquotesingle{}}\SpecialCharTok{\{}\NormalTok{sys}\SpecialCharTok{.}\NormalTok{version}\OperatorTok{=}\SpecialCharTok{\}}\SpecialStringTok{\textquotesingle{}}\NormalTok{)}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
sys.version='3.11.0 | packaged by conda-forge | (main, Jan 14 2023, 12:26:40) [Clang 14.0.6 ]'
\end{verbatim}
Insert text here
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis sagittis
posuere ligula sit amet lacinia. Duis dignissim pellentesque magna,
rhoncus congue sapien finibus mollis. Ut eu sem laoreet, vehicula ipsum
in, convallis erat. Vestibulum magna sem, blandit pulvinar augue sit
amet, auctor malesuada sapien. Nullam faucibus leo eget eros hendrerit,
non laoreet ipsum lacinia. Curabitur cursus diam elit, non tempus ante
volutpat a. Quisque hendrerit blandit purus non fringilla. Integer sit
amet elit viverra ante dapibus semper. Vestibulum viverra rutrum enim,
at luctus enim posuere eu. Orci varius natoque penatibus et magnis dis
parturient montes, nascetur ridiculus mus.
Nunc ac dignissim magna. Vestibulum vitae egestas elit. Proin feugiat
leo quis ante condimentum, eu ornare mauris feugiat. Pellentesque
habitant morbi tristique senectus et netus et malesuada fames ac turpis
egestas. Mauris cursus laoreet ex, dignissim bibendum est posuere
iaculis. Suspendisse et maximus elit. In fringilla gravida ornare.
Aenean id lectus pulvinar, sagittis felis nec, rutrum risus. Nam vel
neque eu arcu blandit fringilla et in quam. Aliquam luctus est sit amet
vestibulum eleifend. Phasellus elementum sagittis molestie. Proin tempor
lorem arcu, at condimentum purus volutpat eu. Fusce et pellentesque
ligula. Pellentesque id tellus at erat luctus fringilla. Suspendisse
potenti.
Etiam maximus accumsan gravida. Maecenas at nunc dignissim, euismod enim
ac, bibendum ipsum. Maecenas vehicula velit in nisl aliquet ultricies.
Nam eget massa interdum, maximus arcu vel, pretium erat. Maecenas sit
amet tempor purus, vitae aliquet nunc. Vivamus cursus urna velit,
eleifend dictum magna laoreet ut. Duis eu erat mollis, blandit magna id,
tincidunt ipsum. Integer massa nibh, commodo eu ex vel, venenatis
efficitur ligula. Integer convallis lacus elit, maximus eleifend lacus
ornare ac. Vestibulum scelerisque viverra urna id lacinia. Vestibulum
ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia
curae; Aenean eget enim at diam bibendum tincidunt eu non purus. Nullam
id magna ultrices, sodales metus viverra, tempus turpis.
Duis ornare ex ac iaculis pretium. Maecenas sagittis odio id erat
pharetra, sit amet consectetur quam sollicitudin. Vivamus pharetra quam
purus, nec sagittis risus pretium at. Nullam feugiat, turpis ac accumsan
interdum, sem tellus blandit neque, id vulputate diam quam semper nisl.
Donec sit amet enim at neque porttitor aliquet. Phasellus facilisis
nulla eget placerat eleifend. Vestibulum non egestas eros, eget lobortis
ipsum. Nulla rutrum massa eget enim aliquam, id porttitor erat luctus.
Nunc sagittis quis eros eu sagittis. Pellentesque dictum, erat at
pellentesque sollicitudin, justo augue pulvinar metus, quis rutrum est
mi nec felis. Vestibulum efficitur mi lorem, at elementum purus
tincidunt a. Aliquam finibus enim magna, vitae pellentesque erat
faucibus at. Nulla mauris tellus, imperdiet id lobortis et, dignissim
condimentum ipsum. Morbi nulla orci, varius at aliquet sed, facilisis id
tortor. Donec ut urna nisi.
Aenean placerat luctus tortor vitae molestie. Nulla at aliquet nulla.
Sed efficitur tellus orci, sed fringilla lectus laoreet eget. Vivamus
maximus quam sit amet arcu dignissim, sed accumsan massa ullamcorper.
Sed iaculis tincidunt feugiat. Nulla in est at nunc ultricies dictum ut
vitae nunc. Aenean convallis vel diam at malesuada. Suspendisse arcu
libero, vehicula tempus ultrices a, placerat sit amet tortor. Sed dictum
id nulla commodo mattis. Aliquam mollis, nunc eu tristique faucibus,
purus lacus tincidunt nulla, ac pretium lorem nunc ut enim. Curabitur
eget mattis nisl, vitae sodales augue. Nam felis massa, bibendum sit
amet nulla vel, vulputate rutrum lacus. Aenean convallis odio pharetra
nulla mattis consequat.
\chapter{Semantic Segmentation (Crop
Mapping)}\label{semantic-segmentation-crop-mapping}
\section{Rice mapping in Bhutan with U-Net using high resolution
satellite
imagery}\label{rice-mapping-in-bhutan-with-u-net-using-high-resolution-satellite-imagery}
\includegraphics{index_files/mediabag/colab_logo_32px.png}
\href{https://colab.research.google.com/github/SERVIR/SERVIR-Applied-Deep-Learning-Book/blob/main/03_Semantic_Segmentation/01__Crop_Mapping/notebooks/Rice_Mapping_Bhutan_2021.ipynb}{Run
in Colab} \includegraphics{index_files/mediabag/GitHub-Mark-32px.png}
\href{https://github.com/SERVIR/SERVIR-Applied-Deep-Learning-Book/blob/main/03_Semantic_Segmentation/01__Crop_Mapping/notebooks/Rice_Mapping_Bhutan_2021.ipynb}{View
on GitHub}
\section{\texorpdfstring{Note: This notebook is meant to be run in
Colab. You can still run this locally, make sure you have
\href{https://developers.google.com/workspace/drive/api/quickstart/python}{Google
Drive API} installed and path adjusted in relevant
places.}{Note: This notebook is meant to be run in Colab. You can still run this locally, make sure you have Google Drive API installed and path adjusted in relevant places.}}\label{note-this-notebook-is-meant-to-be-run-in-colab.-you-can-still-run-this-locally-make-sure-you-have-google-drive-api-installed-and-path-adjusted-in-relevant-places.}
This notebook is also available in this github repo:
\url{https://github.com/SERVIR/servir-aces}. Navigate to the
\texttt{notebook} folder.
\section{Setup environment}\label{setup-environment}
\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{from}\NormalTok{ google.colab }\ImportTok{import}\NormalTok{ drive}
\NormalTok{drive.mount(}\StringTok{"/content/drive"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}
\begin{Shaded}
\begin{Highlighting}[]
\OperatorTok{!}\NormalTok{pip install servir}\OperatorTok{{-}}\NormalTok{aces}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
Collecting servir-aces
Downloading servir_aces-0.0.14-py2.py3-none-any.whl (32 kB)
Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from servir-aces) (1.25.2)
Requirement already satisfied: tensorflow>=2.9.3 in /usr/local/lib/python3.10/dist-packages (from servir-aces) (2.15.0)
Requirement already satisfied: earthengine-api in /usr/local/lib/python3.10/dist-packages (from servir-aces) (0.1.399)
Collecting python-dotenv>=1.0.0 (from servir-aces)
Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (from servir-aces) (3.7.1)
Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (1.4.0)
Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (1.6.3)
Requirement already satisfied: flatbuffers>=23.5.26 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (24.3.25)
Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (0.5.4)
Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (0.2.0)
Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (3.9.0)
Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (18.1.1)
Requirement already satisfied: ml-dtypes~=0.2.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (0.2.0)
Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (3.3.0)
Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (24.0)
Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.20.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (3.20.3)
Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (67.7.2)
Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (1.16.0)
Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (2.4.0)
Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (4.11.0)
Requirement already satisfied: wrapt<1.15,>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (1.14.1)
Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (0.36.0)
Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (1.62.2)
Requirement already satisfied: tensorboard<2.16,>=2.15 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (2.15.2)
Requirement already satisfied: tensorflow-estimator<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (2.15.0)
Requirement already satisfied: keras<2.16,>=2.15.0 in /usr/local/lib/python3.10/dist-packages (from tensorflow>=2.9.3->servir-aces) (2.15.0)
Requirement already satisfied: google-cloud-storage in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (2.8.0)
Requirement already satisfied: google-api-python-client>=1.12.1 in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (2.84.0)
Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (2.27.0)
Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (0.1.1)
Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (0.22.0)
Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from earthengine-api->servir-aces) (2.31.0)
Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (1.2.1)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (0.12.1)
Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (4.51.0)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (1.4.5)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (9.4.0)
Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (3.1.2)
Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib->servir-aces) (2.8.2)
Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from astunparse>=1.6.0->tensorflow>=2.9.3->servir-aces) (0.43.0)
Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client>=1.12.1->earthengine-api->servir-aces) (2.11.1)
Requirement already satisfied: uritemplate<5,>=3.0.1 in /usr/local/lib/python3.10/dist-packages (from google-api-python-client>=1.12.1->earthengine-api->servir-aces) (4.1.1)
Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.4.1->earthengine-api->servir-aces) (5.3.3)
Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.4.1->earthengine-api->servir-aces) (0.4.0)
Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.4.1->earthengine-api->servir-aces) (4.9)
Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (1.2.0)
Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (3.6)
Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (0.7.2)
Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (3.0.2)
Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->earthengine-api->servir-aces) (3.3.2)
Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->earthengine-api->servir-aces) (3.7)
Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->earthengine-api->servir-aces) (2.0.7)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->earthengine-api->servir-aces) (2024.2.2)
Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->earthengine-api->servir-aces) (2.3.3)
Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-storage->earthengine-api->servir-aces) (2.7.0)
Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-api-python-client>=1.12.1->earthengine-api->servir-aces) (1.63.0)
Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (1.3.1)
Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage->earthengine-api->servir-aces) (1.5.0)
Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.4.1->earthengine-api->servir-aces) (0.6.0)
Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (2.1.5)
Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard<2.16,>=2.15->tensorflow>=2.9.3->servir-aces) (3.2.2)
Installing collected packages: python-dotenv, servir-aces
Successfully installed python-dotenv-1.0.1 servir-aces-0.0.14
\end{verbatim}
\begin{Shaded}
\begin{Highlighting}[]
\OperatorTok{!}\NormalTok{git clone https:}\OperatorTok{//}\NormalTok{github.com}\OperatorTok{/}\NormalTok{SERVIR}\OperatorTok{/}\NormalTok{servir}\OperatorTok{{-}}\NormalTok{aces}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
Cloning into 'servir-aces'...
remote: Enumerating objects: 740, done.
remote: Counting objects: 100% (116/116), done.
remote: Compressing objects: 100% (78/78), done.
remote: Total 740 (delta 46), reused 68 (delta 38), pack-reused 624
Receiving objects: 100% (740/740), 5.07 MiB | 16.12 MiB/s, done.
Resolving deltas: 100% (468/468), done.
\end{verbatim}
\subsection{Download datasets}\label{download-datasets}
For this chapter, we have already prepared and exported the training
datasets. They can be found at the google cloud storage and we will use
\texttt{gsutil} to get the dataset in our workspace. The dataset has
\texttt{training}, \texttt{testing}, and \texttt{validation}
subdirectory. Let's start by downloading these datasets in our
workspace.
If you're looking to produce your own datasets, you can follow this
\href{https://colab.research.google.com/drive/1LCFLeSCu969wIW8TD68-j4hfIu7WiRIK?usp=sharing}{notebook}
which was used to produce these training, testing, and validation
datasets provided in this notebook.
\begin{Shaded}
\begin{Highlighting}[]
\OperatorTok{!}\NormalTok{mkdir }\OperatorTok{{-}}\NormalTok{p content}\OperatorTok{/}\NormalTok{datasets}
\end{Highlighting}
\end{Shaded}
\begin{Shaded}
\begin{Highlighting}[]
\OperatorTok{!}\NormalTok{gsutil }\OperatorTok{{-}}\NormalTok{m cp }\OperatorTok{{-}}\NormalTok{r gs:}\OperatorTok{//}\NormalTok{dl}\OperatorTok{{-}}\NormalTok{book}\OperatorTok{/}\NormalTok{chapter}\OperatorTok{{-}}\DecValTok{1}\NormalTok{ content}\OperatorTok{/}\NormalTok{datasets}\OperatorTok{/}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
If you experience problems with multiprocessing on MacOS, they might be related to https://bugs.python.org/issue33725. You can disable multiprocessing by editing your .boto config or by adding the following flag to your command: `-o "GSUtil:parallel_process_count=1"`. Note that multithreading is still available even if you disable multiprocessing.
Copying gs://dl-book/chapter-1/.DS_Store...
Copying gs://dl-book/chapter-1/dnn_planet_wo_indices/testing/testing.tfrecord.gz...
Copying gs://dl-book/chapter-1/dnn_planet_wo_indices/training/training.tfrecord.gz...
Copying gs://dl-book/chapter-1/dnn_planet_wo_indices/validation/validation.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_202100000.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_202100001.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_202100002.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_202100003.tfrecord.gz...
Copying gs://dl-book/chapter-1/models/dnn_v1/config.env...
Copying gs://dl-book/chapter-1/images/image_202100004.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_202100005.tfrecord.gz...
Copying gs://dl-book/chapter-1/images/image_2021mixer.json...
Copying gs://dl-book/chapter-1/models/dnn_v1/aces/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/aces/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/aces/saved_model.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/aces/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/dnn_v1/aces/variables/variables.index...
Copying gs://dl-book/chapter-1/models/dnn_v1/config.json...
Copying gs://dl-book/chapter-1/models/dnn_v1/evaluation.txt...
Copying gs://dl-book/chapter-1/models/dnn_v1/logs/train/events.out.tfevents.1713307528.b5c4c6e407b9.190.2.v2...
Copying gs://dl-book/chapter-1/models/dnn_v1/logs/validation/events.out.tfevents.1713307537.b5c4c6e407b9.190.3.v2...
Copying gs://dl-book/chapter-1/models/dnn_v1/model.png...
Copying gs://dl-book/chapter-1/models/dnn_v1/model.txt...
Copying gs://dl-book/chapter-1/models/dnn_v1/modelCheckpoint/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/modelCheckpoint/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/dnn_v1/modelCheckpoint/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/aces/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/aces/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/model.pkl...
Copying gs://dl-book/chapter-1/models/dnn_v1/modelCheckpoint/variables/variables.index...
Copying gs://dl-book/chapter-1/models/dnn_v1/parameters.txt...
Copying gs://dl-book/chapter-1/models/dnn_v1/prediction/prediction_dnn_v1.TFRecord...
Copying gs://dl-book/chapter-1/models/unet_v1/model.pkl...
Copying gs://dl-book/chapter-1/models/dnn_v1/trained-model/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/modelCheckpoint/saved_model.pb...
==> NOTE: You are downloading one or more large file(s), which would
run significantly faster if you enabled sliced object downloads. This
feature is enabled by default but requires that compiled crcmod be
installed (see "gsutil help crcmod").
Copying gs://dl-book/chapter-1/models/unet_v1/aces/saved_model.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/trained-model/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/trained-model/saved_model.pb...
Copying gs://dl-book/chapter-1/models/dnn_v1/trained-model/variables/variables.index...
Copying gs://dl-book/chapter-1/models/dnn_v1/trained-model/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/dnn_v1/training.png...
Copying gs://dl-book/chapter-1/models/unet_v1/aces/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/unet_v1/model.png...
Copying gs://dl-book/chapter-1/models/unet_v1/logs/train/events.out.tfevents.1713299324.b5c4c6e407b9.190.0.v2...
Copying gs://dl-book/chapter-1/models/unet_v1/config.env...
Copying gs://dl-book/chapter-1/models/unet_v1/evaluation.txt...
Copying gs://dl-book/chapter-1/models/unet_v1/config.json...
Copying gs://dl-book/chapter-1/models/unet_v1/model.txt...
Copying gs://dl-book/chapter-1/models/unet_v1/logs/validation/events.out.tfevents.1713299558.b5c4c6e407b9.190.1.v2...
Copying gs://dl-book/chapter-1/models/unet_v1/aces/variables/variables.index...
Copying gs://dl-book/chapter-1/models/unet_v1/modelCheckpoint/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/modelCheckpoint/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/modelCheckpoint/saved_model.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/modelCheckpoint/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/unet_v1/modelCheckpoint/variables/variables.index...
Copying gs://dl-book/chapter-1/models/unet_v1/prediction/prediction_unet_v1.TFRecord...
Copying gs://dl-book/chapter-1/models/unet_v1/parameters.txt...
Copying gs://dl-book/chapter-1/models/unet_v1/trained-model/fingerprint.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/trained-model/keras_metadata.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/trained-model/saved_model.pb...
Copying gs://dl-book/chapter-1/models/unet_v1/trained-model/variables/variables.data-00000-of-00001...
Copying gs://dl-book/chapter-1/models/unet_v1/trained-model/variables/variables.index...
Copying gs://dl-book/chapter-1/models/unet_v1/training.png...
Copying gs://dl-book/chapter-1/prediction/prediction_dnn_v1.TFRecord...
Copying gs://dl-book/chapter-1/prediction/prediction_unet_v1.TFRecord...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00000-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00001-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00003-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00002-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00004-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00005-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00006-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/training_data/testing_10/testing__256x256-00007-of-00008.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00000-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00001-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00002-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00003-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00004-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00005-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00006-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00007-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00008-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00009-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00010-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00011-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00012-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00013-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00014-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00015-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00016-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00017-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00018-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00019-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00020-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00021-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00022-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00023-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00024-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00025-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00026-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00027-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00028-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00029-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00030-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00031-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00032-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00033-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00034-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00035-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00036-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/testing/testing-00037-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00000-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00001-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00002-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00003-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00004-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00005-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00006-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00007-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00008-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00009-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00010-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00011-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00012-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00013-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00014-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00015-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00016-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00017-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00018-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00019-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00020-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00021-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00022-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00023-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00024-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00025-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00026-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00027-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00028-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00029-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00030-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00031-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00032-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00033-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00034-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00035-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00036-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/training/training-00037-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00000-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00001-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00002-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00003-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00004-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00005-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00006-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00007-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00008-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00009-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00010-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00011-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00012-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00013-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00014-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00015-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00016-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00017-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00018-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00019-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00020-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00021-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00022-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00023-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00024-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00025-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00026-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00027-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00028-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00029-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00030-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00031-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00032-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00033-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00034-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00035-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00036-of-00038.tfrecord.gz...
Copying gs://dl-book/chapter-1/unet_256x256_planet_wo_indices/validation/validation-00037-of-00038.tfrecord.gz...
/ [187/192 files][ 16.3 GiB/ 16.3 GiB] 99% Done 43.1 MiB/s ETA 00:00:00
\end{verbatim}
\subsection{Setup config file
variables}\label{setup-config-file-variables}
Now the repo is downloaded. We will create an environment file file to
place point to our training data and customize parameters for the model.
To do this, we make a copy of the \texttt{.env.example} file provided.
Under the hood, all the configuration provided via the environment file
are parsed as a config object and can be accessed programatically.
Note current version does not expose all the model intracacies through
the environment file but future version may include those depending on
the need.
\begin{Shaded}
\begin{Highlighting}[]
\OperatorTok{!}\NormalTok{cp servir}\OperatorTok{{-}}\NormalTok{aces}\OperatorTok{/}\NormalTok{.env.example servir}\OperatorTok{{-}}\NormalTok{aces}\OperatorTok{/}\NormalTok{config.env}
\end{Highlighting}
\end{Shaded}
Okay, now we have the \texttt{config.env} file, we will use this to
provide our environments and parameters.
Note there are several parameters that can be changed. Let's start by
changing the \texttt{BASEDIR} and \texttt{OUTPUT\_DIR} as below.
\begin{verbatim}
BASEDIR = "/content/"
OUTPUT_DIR = "/content/drive/MyDrive/Colab Notebooks/DL_Book/Chapter_1/output"
\end{verbatim}
We will start by training a \href{https://insert-citation-here}{U-Net}
model using the
\texttt{dl-book/chapter-1/unet\_256x256\_planet\_wo\_indices} dataset
inside the \texttt{dataset} folder for this exercise. Let's go ahead and
change our DATADIR in the \texttt{config.env} file as below.
\begin{verbatim}
DATADIR = "datasets/unet_256x256_planet_wo_indices"
\end{verbatim}
These datasets have RGBN from Planetscope mosiac. Since we are trying to
map the rice fields, we use growing season and pre-growing season
information. Thus, we have 8 optical bands, namely \texttt{red\_before},
\texttt{green\_before}, \texttt{blue\_before}, \texttt{nir\_before},
\texttt{red\_during}, \texttt{green\_during}, \texttt{blue\_during}, and
\texttt{nir\_during}. In adidition, you can use \texttt{USE\_ELEVATION}
and \texttt{USE\_S1} config to include the topographic and radar
information. Since this datasets have toppgraphic and radar features, so
we won't be settting these config values. Similarly, these datasets are
tiled to 256x256 pixels, so let's also change that.
\begin{verbatim}
# For model training, USE_ELEVATION extends FEATURES with "elevation" & "slope"
# USE_S1 extends FEATURES with "vv_asc_before", "vh_asc_before", "vv_asc_during", "vh_asc_during",
# "vv_desc_before", "vh_desc_before", "vv_desc_during", "vh_desc_during"
# In case these are not useful and you have other bands in your training data, you can do set
# USE_ELEVATION and USE_S1 to False and update FEATURES to include needed bands
USE_ELEVATION = False
USE_S1 = False
PATCH_SHAPE = (256, 256)
\end{verbatim}
Next, we need to calculate the size of the traiing, testing and
validation dataset. For this, we know our size before hand. But
\texttt{aces} also provides handful of functions that we can use to
calculate this. See this
\href{https://colab.research.google.com/drive/12WgDI3ptFZHmcfOw89fmPSsDwO-sXIUH?usp=sharing}{notebook}
to learn more about how to do it. We will also change the
\texttt{BATCH\_SIZE} to 32; if you have larger memory available, you can
increase the \texttt{BATCH\_SIZE}. You can run for longer
\texttt{EPOCHS} by changing the \texttt{EPOCHS} paramter; we will keep
it to 5 for now.
\begin{verbatim}
# Sizes of the training and evaluation datasets.
TRAIN_SIZE = 8531
TEST_SIZE = 1222
VAL_SIZE = 2404
BATCH_SIZE = 32
EPOCHS = 30
\end{verbatim}
\subsection{Update the config file
programtically}\label{update-the-config-file-programtically}
We can also make a dictionary so we can change these config settings
programatically.
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{BASEDIR }\OperatorTok{=} \StringTok{"/content/"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{OUTPUT\_DIR }\OperatorTok{=} \StringTok{"/content/drive/MyDrive/Colab Notebooks/DL\_Book/Chapter\_1/output"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{DATADIR }\OperatorTok{=} \StringTok{"datasets/unet\_256x256\_planet\_wo\_indices"} \CommentTok{\# @param \{type:"string"\}}
\CommentTok{\# PATCH\_SHAPE, USE\_ELEVATION, USE\_S1, TRAIN\_SIZE, TEST\_SIZE, VAL\_SIZE}
\CommentTok{\# BATCH\_SIZE, EPOCHS are converted to their appropriate type.}
\NormalTok{USE\_ELEVATION }\OperatorTok{=} \StringTok{"False"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{USE\_S1 }\OperatorTok{=} \StringTok{"False"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{PATCH\_SHAPE }\OperatorTok{=} \StringTok{"(256, 256)"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{TRAIN\_SIZE }\OperatorTok{=} \StringTok{"8531"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{TEST\_SIZE }\OperatorTok{=} \StringTok{"1222"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{VAL\_SIZE }\OperatorTok{=} \StringTok{"2404"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{BATCH\_SIZE }\OperatorTok{=} \StringTok{"32"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{EPOCHS }\OperatorTok{=} \StringTok{"30"} \CommentTok{\# @param \{type:"string"\}}
\NormalTok{MODEL\_DIR\_NAME }\OperatorTok{=} \StringTok{"unet\_v1"} \CommentTok{\# @param \{type:"string"\}}
\end{Highlighting}
\end{Shaded}
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{unet\_config\_settings }\OperatorTok{=}\NormalTok{ \{}
\StringTok{"BASEDIR"}\NormalTok{ : BASEDIR,}
\StringTok{"OUTPUT\_DIR"}\NormalTok{: OUTPUT\_DIR,}
\StringTok{"DATADIR"}\NormalTok{: DATADIR,}
\StringTok{"USE\_ELEVATION"}\NormalTok{: USE\_ELEVATION,}
\StringTok{"USE\_S1"}\NormalTok{: USE\_S1,}
\StringTok{"PATCH\_SHAPE"}\NormalTok{: PATCH\_SHAPE,}
\StringTok{"TRAIN\_SIZE"}\NormalTok{: TRAIN\_SIZE,}
\StringTok{"TEST\_SIZE"}\NormalTok{: TEST\_SIZE,}
\StringTok{"VAL\_SIZE"}\NormalTok{: VAL\_SIZE,}
\StringTok{"BATCH\_SIZE"}\NormalTok{: BATCH\_SIZE,}
\StringTok{"EPOCHS"}\NormalTok{: EPOCHS,}
\StringTok{"MODEL\_DIR\_NAME"}\NormalTok{: MODEL\_DIR\_NAME,}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}
\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{import}\NormalTok{ dotenv}
\NormalTok{config\_file }\OperatorTok{=} \StringTok{"servir{-}aces/config.env"}
\ControlFlowTok{for}\NormalTok{ config\_key }\KeywordTok{in}\NormalTok{ unet\_config\_settings:}
\NormalTok{ dotenv.set\_key(dotenv\_path}\OperatorTok{=}\NormalTok{config\_file,}
\NormalTok{ key\_to\_set}\OperatorTok{=}\NormalTok{config\_key,}
\NormalTok{ value\_to\_set}\OperatorTok{=}\NormalTok{unet\_config\_settings[config\_key]}
\NormalTok{ )}
\end{Highlighting}
\end{Shaded}
\section{U-Net Model}\label{u-net-model}
\subsection{Load config file
variables}\label{load-config-file-variables}
\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{from}\NormalTok{ aces }\ImportTok{import}\NormalTok{ Config, DataProcessor, ModelTrainer, EEUtils}
\end{Highlighting}
\end{Shaded}
Let's load our config file through the \texttt{Config} class.
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{unet\_config }\OperatorTok{=}\NormalTok{ Config(config\_file}\OperatorTok{=}\NormalTok{config\_file)}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
BASEDIR: /content
DATADIR: /content/datasets/unet_256x256_planet_wo_indices
using features: ['red_before', 'green_before', 'blue_before', 'nir_before', 'red_during', 'green_during', 'blue_during', 'nir_during']
using labels: ['class']
\end{verbatim}
Most of the config in the \texttt{config.env} is now available via the
config instance. Let's check few of them here.
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{unet\_config.TRAINING\_DIR, unet\_config.OUTPUT\_DIR, unet\_config.BATCH\_SIZE, unet\_config.TRAIN\_SIZE}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
(PosixPath('/content/datasets/unet_256x256_planet_wo_indices/training'),
PosixPath('/content/drive/MyDrive/Colab Notebooks/DL_Book/Chapter_1/output'),
32,
8531)
\end{verbatim}
\subsection{\texorpdfstring{Load \texttt{ModelTrainer}
class}{Load ModelTrainer class}}\label{load-modeltrainer-class}
Next, let's make an instance of the \texttt{ModelTrainer} object. The
\texttt{ModelTrainer} class provides various tools for training,
buidling, compiling, and running specified deep learning models.
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{unet\_model\_trainer }\OperatorTok{=}\NormalTok{ ModelTrainer(unet\_config, seed}\OperatorTok{=}\DecValTok{42}\NormalTok{)}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
Using seed: 42
\end{verbatim}
\subsection{Train and Save U-Net
model}\label{train-and-save-u-net-model}
\href{https://servir.github.io/servir-aces/model_trainer/}{\texttt{ModelTrainer}}
class provides various functionality. We will use \texttt{train\_model}
function that helps to train the model using the provided configuration
settings.
This method performs the following steps: - Configures memory growth for
TensorFlow. - Creates TensorFlow datasets for training, testing, and
validation. - Builds and compiles the model. - Prepares the output
directory for saving models and results. - Starts the training process.
- Evaluates and prints validation metrics. - Saves training parameters,
plots, and models.
\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{unet\_model\_trainer.train\_model()}
\end{Highlighting}
\end{Shaded}
\begin{verbatim}
****************************************************************************
****************************** Clear Session... ****************************
****************************************************************************
****************************** Configure memory growth... ************************
> Found 1 GPUs
****************************************************************************
****************************** creating datasets... ************************
Loading dataset from /content/datasets/unet_256x256_planet_wo_indices/training/*
randomly transforming data
Loading dataset from /content/datasets/unet_256x256_planet_wo_indices/validation/*
Loading dataset from /content/datasets/unet_256x256_planet_wo_indices/testing/*
Printing dataset info:
Training
inputs: float32 (32, 256, 256, 8)
tf.Tensor(
[[[[0.073075 0.063275 0.0411 ... 0.050625 0.0274 0.23925 ]
[0.084775 0.067375 0.047025 ... 0.057675 0.032075 0.242375]
[0.083625 0.068575 0.045075 ... 0.059275 0.0332 0.2409 ]
...
[0.0702 0.06825 0.04495 ... 0.055025 0.028325 0.26305 ]
[0.064475 0.066 0.043575 ... 0.0524 0.027075 0.26705 ]
[0.0676 0.06355 0.04535 ... 0.05375 0.02875 0.263275]]
[[0.071475 0.062225 0.0388 ... 0.0496 0.025375 0.24155 ]
[0.07815 0.065025 0.044225 ... 0.0545 0.02905 0.24175 ]
[0.086025 0.069125 0.046175 ... 0.05855 0.0326 0.2355 ]
...
[0.060775 0.0627 0.041875 ... 0.051575 0.029725 0.267475]
[0.061375 0.06225 0.04225 ... 0.0513 0.02685 0.268375]
[0.06845 0.064075 0.043925 ... 0.052925 0.028575 0.267975]]
[[0.0677 0.0605 0.038625 ... 0.04835 0.024825 0.236075]
[0.078375 0.0629 0.04215 ... 0.0524 0.02855 0.237375]
[0.0857 0.065725 0.04635 ... 0.05705 0.030975 0.235375]
...
[0.07 0.062775 0.04485 ... 0.053425 0.0292 0.27015 ]
[0.0607 0.060675 0.041175 ... 0.053075 0.026275 0.27025 ]
[0.068 0.0667 0.045375 ... 0.055475 0.029375 0.262725]]
...
[[0.083525 0.06785 0.044125 ... 0.06365 0.0331 0.234825]
[0.097825 0.07235 0.047925 ... 0.06675 0.03365 0.2363 ]
[0.1092 0.082125 0.05385 ... 0.072125 0.036225 0.2486 ]
...
[0.08935 0.088725 0.067575 ... 0.079675 0.042425 0.38085 ]
[0.093725 0.0875 0.06355 ... 0.07565 0.04185 0.344525]
[0.0937 0.089675 0.066775 ... 0.07465 0.043025 0.330925]]
[[0.0893 0.0732 0.04715 ... 0.065 0.0351 0.233525]
[0.091325 0.073425 0.047475 ... 0.0653 0.032675 0.238325]
[0.096775 0.07645 0.051625 ... 0.06875 0.0344 0.252825]
...
[0.0836 0.084875 0.061975 ... 0.07825 0.042875 0.38785 ]
[0.08865 0.083825 0.060675 ... 0.0765 0.042525 0.3522 ]
[0.0909 0.084475 0.061975 ... 0.0769 0.043275 0.342625]]
[[0.092075 0.078 0.050925 ... 0.06565 0.03555 0.235275]
[0.0805 0.0705 0.043325 ... 0.063925 0.03215 0.243875]
[0.086925 0.074025 0.0495 ... 0.067475 0.03345 0.26095 ]
...
[0.081075 0.078725 0.056425 ... 0.07505 0.0398 0.37805 ]
[0.0865 0.079375 0.05845 ... 0.076175 0.0439 0.3619 ]
[0.0886 0.077775 0.057725 ... 0.076175 0.042825 0.3439 ]]]