CO432/misc

Author: RetroCraft

CO432/notes.pdf

@@ -272,31 +272,31 @@ \section{Entropy as optimal lossless data compression}
 by amortizing over longer batches of the string.
 
 \begin{sol}[batching]
-  For $\rv Y$ defined on $[n]$ equal to $i$ with probability $q_i$,
-  define the random variable $\rv Y^{(k)}$ on $[n]^k$
+  For $\Y$ defined on $[n]$ equal to $i$ with probability $q_i$,
+  define the random variable $\Y^{(k)}$ on $[n]^k$
   equal to the string $i_1\cdots i_k$ with probability $q_{i_1}\cdots q_{i_k}$.
-  That is, $\rv Y^{(k)}$ models $k$ independent samples of $\rv Y$.
+  That is, $\Y^{(k)}$ models $k$ independent samples of $\Y$.
 
-  Apply the Shannon--Fano code to $\rv Y^{(k)}$
+  Apply the Shannon--Fano code to $\Y^{(k)}$
   to get an encoding of $[n]^k$ as bitstrings of expected length $\ell$
-  satisfying $H(\rv Y^{(k)}) \leq \ell \leq H(\rv Y^{(k)}) + 1$.
+  satisfying $H(\Y^{(k)}) \leq \ell \leq H(\Y^{(k)}) + 1$.
   \begin{align*}
-    H(\rv Y^{(k)}) & = \E_{i_1\cdots i_k \sim \rv Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_1}\cdots q_{i_k}}] \tag{by def'n}                       \\
-                   & = \E_{i_1\cdots i_k \sim \rv Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_1}} + \dotsb + \log_2\frac{1}{q_{i_k}}] \tag{log rules} \\
-                   & = \sum_{j=1}^k \E_{i_1\cdots i_k \sim \rv Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_j}}] \tag{linearity of expectation}        \\
-                   & = \sum_{j=1}^k \E_{i \sim \rv Y}\qty[\log_2 \frac{1}{q_{i}}] \tag{$q_{i_j}$ only depends on one character}             \\
-                   & = kH(\rv Y) \tag{by def'n, no $j$-dependence in sum}
+    H(\Y^{(k)}) & = \E_{i_1\cdots i_k \sim \Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_1}\cdots q_{i_k}}] \tag{by def'n}                       \\
+                   & = \E_{i_1\cdots i_k \sim \Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_1}} + \dotsb + \log_2\frac{1}{q_{i_k}}] \tag{log rules} \\
+                   & = \sum_{j=1}^k \E_{i_1\cdots i_k \sim \Y^{(k)}}\qty[\log_2 \frac{1}{q_{i_j}}] \tag{linearity of expectation}        \\
+                   & = \sum_{j=1}^k \E_{i \sim \Y}\qty[\log_2 \frac{1}{q_{i}}] \tag{$q_{i_j}$ only depends on one character}             \\
+                   & = kH(\Y) \tag{by def'n, no $j$-dependence in sum}
   \end{align*}
   For every $k$ symbols, we use $\ell$ bits, i.e., $\frac{\ell}{k}$ bits per symbol.
   From the Shannon--Faro bound, we have
   \begin{align*}
-    \frac{H(\rv Y^{(k)})}{k} & \leq \frac{\ell}{k} < \frac{H(\rv Y^{(k)})}{k} + \frac{1}{k} \\
-    H(\rv Y)                 & \leq \frac{\ell}{k} < H(\rv Y) + \frac{1}{k}
+    \frac{H(\Y^{(k)})}{k} & \leq \frac{\ell}{k} < \frac{H(\Y^{(k)})}{k} + \frac{1}{k} \\
+    H(\Y)                 & \leq \frac{\ell}{k} < H(\Y) + \frac{1}{k}
   \end{align*}
-  Then, we have a code for $\rv Y$ bounded by
-  $[H(\rv Y), H(\rv Y) + \frac{1}{k})$.
+  Then, we have a code for $\Y$ bounded by
+  $[H(\Y), H(\Y) + \frac{1}{k})$.
 
-  Taking a limit of some sort, we can say that we need $H(\rv Y) + o(1)$ bits.
+  Taking a limit of some sort, we can say that we need $H(\Y) + o(1)$ bits.
 \end{sol}
 
 \begin{defn*}[relative entropy]
@@ -457,8 +457,8 @@ \chapter{Applications of KL divergence}
 
   That is, $H(p) = p\log_2\frac1p + (1-p)\log_2\frac{1}{1-p}$.
 
-  Likewise, write $\D q p$ to be $\D{\rv Y}{\X}$
-  where $\rv Y \sim \Bern(q)$.
+  Likewise, write $\D q p$ to be $\D{\Y}{\X}$
+  where $\Y \sim \Bern(q)$.
 \end{notation}
 
 Recall Sterling's approximation (which we have used before):
@@ -601,12 +601,12 @@ \section{Rejection sampling}
   Suppose $\X = \begin{cases}
     0 & p=\frac12 \\
     1 & p=\frac12
-  \end{cases}$ and $\rv Y = \begin{cases}
+  \end{cases}$ and $\Y = \begin{cases}
     0 & p=\frac14 \\
     1 & p=\frac34
   \end{cases}$.
 
-  How can we sample $\rv Y$ using $\X$?
+  How can we sample $\Y$ using $\X$?
 \end{example}
 \begin{sol}[naive]
   Take \iid $\X_1$ and $\X_2$.

CO432/notes.tex

@@ -1,5 +1,8 @@
 \newcommand{\bits}[1]{\ensuremath{\{0,1\}^{#1}}}
 \newcommand{\X}{\rv{X}}
+\newcommand{\Y}{\rv{Y}}
+\newcommand{\XX}{\sv{X}}
+\newcommand{\YY}{\sv{Y}}
 \newcommand{\D}[2]{D(#1 \parallel #2)}
 \DeclareMathOperator{\Bern}{Bernoulli}
 \newcommand{\iid}{\textsc{iid}\xspace}

latex/agony-co432.tex

@@ -64,7 +64,7 @@
 \usepackage[titles]{tocloft}
 \addto\captionsenglish{\renewcommand{\contentsname}{\@title}}
 \usepackage{titlesec}
-\usepackage{multicol,collcell}
+\usepackage{multirow,multicol,collcell}
 \usepackage[dvipsnames,table]{xcolor}
 \usepackage{array} % for \newcolumntype macro
 
@@ -98,6 +98,7 @@
 % \catcode`^=7 \catcode`_=8
 
 % Figures
+\RequirePackage{float}
 \ifagony@tikz
   \RequirePackage{tikz,pgfplots,transparent,annotate-equations}
   \pgfplotsset{compat=1.15}
@@ -206,14 +207,15 @@
 \newcommand{\rv}{\mathsf} % random variable => sans-serif
 \newcommand{\vv}{\mathsf} % algorithm variable => sans-serif
 \newcommand{\xx}{\mathtt} % hex literal => typewriter
+\newcommand{\sv}{\mathcal} % set variable => caligraphic
 
 % Sets
-\newcommand{\N}{\ensuremath{\mathbb{N}}}
-\newcommand{\Z}{\ensuremath{\mathbb{Z}}}
-\newcommand{\Q}{\ensuremath{\mathbb{Q}}}
-\newcommand{\R}{\ensuremath{\mathbb{R}}}
-\newcommand{\C}{\ensuremath{\mathbb{C}}}
-\newcommand{\F}{\ensuremath{\mathbb{F}}}
+\newcommand{\N}{\ensuremath{\bb{N}}}
+\newcommand{\Z}{\ensuremath{\bb{Z}}}
+\newcommand{\Q}{\ensuremath{\bb{Q}}}
+\newcommand{\R}{\ensuremath{\bb{R}}}
+\newcommand{\C}{\ensuremath{\bb{C}}}
+\newcommand{\F}{\ensuremath{\bb{F}}}
 
 % Functions
 \newcommand{\fn}{\operatorname}