\documentclass[11pt]{article}
\input{headers}

\usepackage{fancyhdr}   
\pagestyle{fancy}      
\lhead{}               
\rhead{Name: Hemanta K. Maji} %%% <-- YOUR NAME HERE

\usepackage[strict]{changepage}  
\newcommand{\nextoddpage}{\checkoddpage\ifoddpage{\ \newpage\ \newpage}\else{\ \newpage}\fi}  


\begin{document}

\title{Homework 1\\(120 points)}

\date{}

\maketitle 

\thispagestyle{fancy}  
\pagestyle{fancy}  

    




\begin{enumerate}

%%% Problem 1 
\item {\bfseries (20 points) Upper-bound on Entropy.} 
  Let $\Omega = \{1,2,\dotsc,N\}$. 
  Suppose $\X$ is a random variable over the sample space $\Omega$. 
  For shorthand, let us use $x_i = \probX{\X=i}$, for each $i\in\Omega$. 
  The entropy of the random variable $\X$ is defined to be the following function. 
  $$H(\X) \defeq \sum_{i\in\Omega} -p_i\log p_i$$
  
  Use Jensen's inequality on the function $f(x) = \log x$ to prove that the following inequality.
    $$H(\X)\leq \log N$$
  Equality holds if and only if we have $p_1 = p_2 = \dotsi = p_N$. 
  
  
\nextoddpage
\item {\bfseries (20 points) Log-sum Inequality.} 
  Let $\{a_1,\dotsc,a_N\}$ and $\{b_1,\dotsc,b_N\}$ be two sets of non-negative real numbers. 
  Use Jensen's inequality to prove the following inequality. 
    $$\sum_{i=1}^N a_i\log\frac{a_i}{b_i} \geq a\log\frac ab,$$
  where $a = \sum_{i=1}^n a_i$ and $b=\sum_{i=1}^N b_i$. 
  Moreover, equality holds if and only if $\frac{a_i}{b_i}$ is equal for all $i\in\{1,\dotsc,N\}$. 
  
  
\nextoddpage
\item {\bfseries (20 points) Approximating Square-root.}
  Our goal is to find a (meaningful and tight) upper-bound for $f(x) = (1-x)^{1/2}$ using a quadratic function of the form 
    $$g(x) = 1 -\alpha x - \beta x^2$$
  Use the Lagrange form of the Taylor's remainder theorem on $f(x)$ around $x=0$ to obtain the function $g(x)$. 
  
  
\nextoddpage
\item {\bfseries (15+20+5 points) Coupon Collector Problem.} 
  We shall show the following result.
  ``The expected numbers of balls that we need to throw such that every bin has at least one ball is (roughly) $n\log n$.''  
  \begin{enumerate}
  \item Let $i\in\{1,\dotsc,n-1\}$. 
    Suppose a few balls have already been thrown and we have the guarantee that there are exactly $(i-1)$ bins with $>0$ load. 
    Let $\N_i$ be the random variable over the sample space $\{1,2,\dotsc\}$ that represents the minimum number of \underline{additional balls} thrown so that (a total of) $i$ bins have $>0$ load. 
    For any $j\in\{1,2,\dotsc\}$, prove that 
      $$\probX{\N_i = j} = \left(1-\frac{i-1}n\right)\left(\frac{i-1}n\right)^{j-1}$$
  \item Prove that 
    $$\EX{\N_i}= \frac{n}{n-i+1}$$
  \item Denote $\N = \N_1 + \N_2 + \dotsi + \N_n$. 
    So, the random variable $\N$ represents the number of balls so that all bins have load $>0$. 
    Prove that 
    $$\EX{\N} = n\left(1 + \frac12 + \dotsi + \frac1n\right)$$ 
    This quantity is roughly $n\log n$. 
    So, we have proven the bound on the coupon collector problem, which was our goal! 
  \end{enumerate} 
%  Suppose we are performing the following experiment. 
%  We throw infinitely many balls into $n$ bins uniformly and independently at random. 
%  Let $\N_i$ represent the number of thrown balls when, for the first time, $i$ different bins have at least one ball in them. 
%  
%  \begin{boxedalgo}
%  {\em Example.} 
%  Let me explain this using an example. 
%  Suppose $n=3$ and the bins where the balls went are 
%  $$2, 2, 1, 1, 2, 3, 1, 2, \dotsc $$
%  
%  \begin{itemize}
%  \item $\N_1=1$: Because the first ball went to bin 2. 
%    So, the first time we see 1 bin having at least one ball is after the first ball has been thrown. 
%    Note that this is true always! 
%  \item $\N_2=3$: Because the first ball went to bin 2, second ball went to bin 1, and the third ball went to bin 1. 
%    So, the first time we see 2 bins having at least one ball is after the third ball has been thrown. 
%  \item $\N_3=6$: Because, the first 5 balls went to two bins. 
%    So, the first time we see 3 bins having at least one ball is after the 6-th ball has been thrown. 
%  \end{itemize} 
%  \end{boxedalgo} 
%  
%  Let us understand this problem as a coupon collector problem. 
%  The bins are the coupons. 
%  The random variable $\N_i$ represents the first time you see $i$ distinct bins (or, the coupons) 
%  The coupon collector's  $\EX{\N_1+\N_2+\dotsi+\N_n}$. 
  





\nextoddpage
\item {\bfseries (7+7+6 points) Poisson Distribution.}
  We can also approach the Coupon Collector problem using the Poisson Approximation Theorem. 
  To get us started towards that, let us consider the following problem. 
  
  Suppose we have $m$ balls and $n$ bins. 
  Let $\X$ be the Poisson distribution with mean $\mu=m/n$. 
  That is, for all $i\in\{0,1,2,\dotsc\}$, we have
    $$\probX{\X=i} = \exp(-\mu) \frac{\mu^i}{i!}$$
    
  \begin{enumerate}
  \item Find the following probability 
    $$\probX{\X \geq 1}$$
  \item Suppose $(\X\p1,\dotsc,X\p n)$ be $n$ independent samples of the random variable $\X$. 
    Find the probability 
    $$\probX{\X\p1\geq1,\dotsc,\X\p n\geq 1}$$
  \item Substituting $m=n\log n$, compute the probability 
    $$\probX{\X\p1\geq1,\dotsc,\X\p n\geq 1}$$
  \end{enumerate}
  It is left as an exercise to think on how to proceed from here to prove the bounds on the coupon collector problem. 







 
 



\end{enumerate}









\end{document}
