\documentclass[11pt]{article}
\input{headers01}
\lhead{CS 585-Theoretical CS Toolkit, SPRING 2023}
\rhead{Name: Write your name} %%% <-- REPLACE Hemanta K. Maji WITH YOUR NAME HERE
\usepackage[strict]{changepage}
\newcommand{\nextoddpage}{\checkoddpage\ifoddpage{\ \newpage\ \newpage}\else{\ \newpage}\fi}
\begin{document}
\title{Homework 1}
\date{}
\maketitle
\thispagestyle{fancy}
\pagestyle{fancy}
{\bfseries Collaborators :} \newline
% ENTER THEIR NAMES HERE
\begin{enumerate}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 1 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Upper-bound on Entropy.} (20 points)
Let $\Omega=\{1,2,\dotsc,N\}$.
Suppose $\X$ is a random variable over the sample space $\Omega$.
For shorthand, let $p_i = \probX{\X=i}$, for each $i\in\Omega$.
The random variable $\ X$'s entropy is defined as the following function.
$$ \HH{\X} \defeq \sum_{i\in\Omega} -p_i\cdot\ln p_i.$$
Use Jensen's inequality on the function $f(t)=\ln t$ to prove the following inequality.
$$ \HH{\X} \leq \ln N.$$
Furthermore, equality holds if and only if $\X$ is the uniform distribution over $\Omega$.
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 2 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Log-sum Inequality.} (27=22+5 points)
\begin{enumerate}
\item Let $\{a_1,\dotsc,a_N\}$ and $\{b_1,\dotsc,b_N\}$ be two sets of positive real numbers.
Use Jensen's inequality to prove the following inequality.
$$\sum_{i=1}^N a_i \ln \frac{a_i}{b_i} \geq A\ln\frac AB,$$
where $A \defeq \sum_{i=1}^Na_i$ and $B \defeq \sum_{i=1}^Nb_i$.
Furthermore, equality holds if and only if $a_i/b_i$ is identical for all $i\in\{1,\dotsc,N\}$.
\newline
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
\item Let $\cX$ be a finite set and $P:\cX\to [0,1]$ and $Q:\cX\to [0,1]$ be two probability distributions on $\cX$ such that for any $x\in \cX$, $Q(x)\not=0$. The relative entropy from $Q$ to $P$ is defined as follows:
$$\kl{P}{Q}\defeq \sum_{x\in \cX} P(x)\log{\frac{P(x)}{Q(x)}}.$$
Show that for any $P$ and $Q$, it holds that $\kl{P}{Q}\geq 0$. Moreover, state when $\kl{P}{Q}=0$.
\end{enumerate}
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 3 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Approximating Square-root.} (20 points)
Our objective is to find a (meaningful and tight) lower bound for the function $f(x) = (1-x)^{-1/2}$ when $x\in [0,1)$ using a quadratic function of the form
$$g(x) = 1 +\alpha x+\beta x^2.$$
Use the Lagrange form of Taylor's remainder theorem on $f(x)$ around $x=0$ to obtain the function $g(x)$.
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 4 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Lower-bounding Logarithm Function.} (20 points)
By Taylor's Theorem, we have seen that the following upper bound is true.
\begin{boxedalgo}
For all $\eps\in[0,1)$ and integer $k\geq 1$, we have
$$\ln(1-\eps) \leq -\eps - \frac{\eps^2}{2} - \dotsi - \frac{\eps^k}k$$
\end{boxedalgo}
We want a tight lower bound for $\ln(1-\eps)$.
Prove the following lower-bound.
\begin{boxedalgo}
For all $\eps\in[0,1/2]$ and integer $k\geq 1$, we have
$$\ln(1-\eps) \geq \left(-\eps - \frac{\eps^2}{2} - \dotsi - \frac{\eps^k}k\right) - \frac{\eps^k}{k}$$
\end{boxedalgo}
(For visualization of this bound, follow this \href{https://www.desmos.com/calculator/o3iwil80fp}{link})
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 5 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Using Stirling Approximation.} (23 points)
Suppose we have a coin that outputs heads with probability $p$ and outputs tails with probability $q=1-p$.
We toss this coin (independently) $n$ times and record each outcome.
Let $\H$ be the random variable representing the number of heads in this experiment.
Note that the following expression gives the probability that we get a total of $k$ heads.
$$\probX{\H=k} = \choose n k p^k q^{n-k}$$
We will prove upper and lower bounds for this problem, assuming $k \geq pn$.
Define $p'\defeq k/n=(p+\eps)$.
Let $P$ and $P'$ be two probability distributions on the set $\cX=\{\text{tails},\text{heads}\}$ such that
$\pr(P=\text{heads})=p$ and $\pr(P'=\text{heads})=p'$.
Using the (Robbin's form of) Stirling approximation in the lecture notes, prove the following bound.
$$ \frac1{\sqrt{8n p'(1-p')}}\exp\left(-n\kl{P'}{P}\right) \leq \probX{\H=k} \leq \frac1{\sqrt{2\pi n p'(1-p')}}\exp\left(-n\kl{P'}{P}\right),$$
where $\kl{P'}{P}$ is the relative entropy from $P$ to $P'$ defined in question 3.
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 6 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item \textbf{Computing a limit.} (20 points) Compute the following limit
$$\lim_{n\to \infty} \sum_{j=1}^n\frac{\sqrt{4n^2-j^2}}{n^2}.$$
\noindent{\bfseries Solution.}\newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\newpage
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% %%%%%%%%%%%% PROBLEM 7 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\item \textbf{Birthday Bound.} (20 points)
Intuitively, we want to claim that the following two expressions are ``good approximations'' of each other.
$$ f_n(t) \defeq \left(1-\frac1n\right) \left(1-\frac2n\right) \dotsi \left( 1-\frac{t-1}{n}\right)$$
And
$$ g_n(t) \defeq \exp\left(-\frac{t^2}{2n}\right)$$
To formalize this intuition, write the mathematical theorems (and then prove them) when $t=o(n^{2/3})$.
Hint: You may find the following inequalities helpful.
\begin{enumerate}
\item $\ln(1-x)\leq -x$, for $x\in[0,1)$, and
\item $\ln(1-x) \geq -x-x^2$, for $x\in[0,1/2]$ (you already prove this identity earlier).
\end{enumerate}
\newpage
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PROBLEM 8 %%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\item {\bfseries Tight Estimation: Central Binomial Coefficient.} (Extra credit: 15 points)
We will learn a new powerful technique to prove tight inequalities.
As a representative example, we will estimate the central binomial coefficient.
For positive integer $n$, we will prove that
$$L_n \leq \binom{2n}{n} \leq U_n ,$$
where
\begin{align*}
L_n &\defeq \frac{4^n}{\sqrt{\pi\left(n + \frac14 + \frac1{32n}\right)}} &&&
U_n &\defeq \frac{4^n}{\sqrt{\pi\left(n + \frac14 + \frac1{46n}\right)}}.
\end{align*}
To prove these bounds, we will use the following general strategy.
\begin{enumerate}
\item Define the following two sequences
\begin{align*}
\left\{a_n \defeq \binom{2n}{n}/U_n \right\}_{n} &&& \left\{b_n \defeq \binom{2n}{n}/L_n \right\}_{n}
\end{align*}
\item {\em Prove the following limit.}
$$ \lim_{n\to\infty} a_n = \lim_{n\to\infty} b_n = \lim_{n\to\infty} \frac{\binom{2n}{n}}{4^n/\sqrt{\pi n}} = 1, $$
using the Stirling approximation $n! \sim \sqrt{2\pi n}\cdot(n/\mathrm e)^n$.
\item {\em Prove $\left\{a_n\right\}_n$ is an increasing sequence.}
\item {\em From (b) and (c), conclude that $a_n\leq 1$, implying $\binom{2n}{n} \leq U_n$.}
\item {\em Prove $\left\{b_n\right\}_n$ is a decreasing sequence.}
\item {\em From (b) and (e), conclude that $b_n\geq 1$, implying $\binom{2n}{n} \geq L_n$.}
\end{enumerate}
\begin{boxedalgo}
{\em Remark: What did we achieve from this exercise? }
We started from the asymptotic estimate $\binom{2n}{n} \sim 4^n/\sqrt{\pi n}$.
From this asymptotic estimate, we obtained explicit upper and lower bounds.
We learned a powerful general technique to translate asymptotic estimates into explicit upper and lower bounds automatically.
\end{boxedalgo}
{\bfseries Solution.} \newline
\ %%% <-- ERASE THIS LINE AND WRITE YOUR SOLUTION HERE
\end{enumerate}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%% PLEASE LIST COLLABORATORS BELOW %%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\end{document}