Appendix01 Proof of Simple Linear Regression

1. $\left\{ \begin{array}{ll} \hat{\beta}_{1} =\frac{\sum_{i=1}^{n}(x_{i}-\bar{x})(y_{i}-\bar{y})}{\sum_{i=1}^{n}(x-\bar{x})^{2}} =\frac{\sum_{i=1}^{n}x_{i}y_{i}-n\cdot{\bar{x}\bar{y}}}{\sum_{i=1}^{n}x_{i}^{2}-n\cdot{\bar{x}^2}} =\frac{S_{xy}}{S_{x}}=\frac{cov(x,y)}{var(x)}\\ \hat{\beta}_{0}=\bar{y}-\hat{\beta}_{1}\bar{x} \end{array} \right.$

   $\begin{array}{ll} \because & {\rm min}\bigg(\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_i)^2 \bigg) \\ \therefore & {\rm create\ a\ function}\\ & f(\beta)=\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_i)^2\\ \therefore & {\rm we\ wish\ the\ partial\ derivative\ of\ } f(\beta)\ {\rm equal\ to\ zero,\ as}\\ & \frac{\partial{f(\beta)}}{\beta_{0}} =\frac{\partial{f(\beta)}}{\beta_{1}}=0\\ \because & \left\{ \begin{array}{ll} \frac{\partial{f(\beta)}}{\partial{\beta_0}} =2\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})\times(-1) =2\sum_{i=1}^{n}(\beta_{1}x_{i}+\beta_{0}-y_{i}) =2\bigg(\beta_{1}\cdot\sum_{i=1}^{n}x_{i}+n\cdot\beta_{0}-\sum_{i=1}^{n}{y_i} \bigg)\\ \frac{\partial{f(\beta)}}{\partial{\beta_1}}=2\sum_{i=1}^{n}(y_{i}-\beta_{0}-\beta_{1}x_{i})\times(-x_{i})=2\sum_{i=1}^{n}(\beta_{1}x_{i}^2+\beta_{0}x_{i}-x_{i}y_{i}) \end{array} \right.\\ \therefore & \left\{ \begin{array}{ll} \frac{\partial{f(\beta)}}{\partial{\beta_{0}}} =2\bigg(\beta_{1}\cdot\sum_{1=1}^{n}x_{i}+n\cdot\beta_{0}-\sum_{i=1}^{n}y_{i}\bigg)=n\beta_{1}\cdot\bar{x}+n\beta_{0}-n\cdot\bar{y} =0\\ \frac{\partial{f(\beta)}}{\partial{\beta_{1}}}=2\sum_{i=1}^{n}(\beta_{1}x_{i}^{2}+\beta_0x_{i}-x_{i}y_{i})=0 \end{array} \right.\\ \therefore & \left\{ \begin{array}{ll} n\beta_{1}\cdot\bar{x}+n\beta_{0}-n\cdot\bar{y} =\beta_{1}\cdot\bar{x}+\beta_{0}-\bar{y} =0\\ 2\sum_{i=1}^{n}(\beta_{1}x_{i}^{2}+\beta_0x_{i}-x_{i}y_{i})=0 \end{array} \right.\\ \therefore & \left\{ \begin{array}{ll} \beta_{0}=\bar{y}-\beta_{1}\cdot\bar{x}\\ 2\sum_{i=1}^{n}(\beta_{1}x_{i}^{2}+\beta_0x_{i}-x_{i}y_{i})=0 \end{array} \right.\\ \therefore & 2\sum_{i=1}^{n}(\beta_{1}x_{i}^{2}+(\bar{y}-\beta_{1}\bar{x})\cdot x_{i}-x_{i}y_{i}) \\ & =2\beta_{1}\sum_{i=1}^{n}x_{i}^{2}+2(\bar{y}-\beta_{1}\bar{x})\sum_{i=1}^{n}x_{i}-2\sum_{i=1}^{n}{x_{i}y_{i}}\\ & =2\beta_{1}\cdot\bigg(\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^2\bigg)+2n\cdot\bar{x}\bar{y}-2\sum_{i=1}^{n}{x_{i}y_{i}}\\ & =0\\ \because & 2\beta_{1}\cdot\bigg(\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^2\bigg)+2n\cdot\bar{x}\bar{y}-2\sum_{i=1}^{n}{x_{i}y_{i}}=0 \\ \therefore & \beta_{1}=\frac{2\sum_{i=1}^{n}x_{i}y_{i}-2n\cdot\bar{x}\bar{y}}{2(\sum_{i=1}^n-n\cdot\bar{x}^2)}\\ \therefore & \beta_{1}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-n\cdot\bar{x}\bar{y}}{\sum_{i=1}^n-n\cdot\bar{x}^2}=\frac{cov(x,y)}{cov(x,x)}=\frac{cov(x,y)}{var(x)}\\ \because & \beta_{0}=\bar{y}-\bar{x}\beta_{1}\\ \therefore & \beta_{0}=\bar{y}-\bar{x}\cdot\frac{\sum_{i=1}^{n}x_{i}y_{i}-n\cdot\bar{x}\bar{y}}{\sum_{i=1}^n-n\cdot\bar{x}^2} \end{array}$

2. $Var(\bar{y})=\frac{\sigma^{2}}{n}$

   $\begin{array}{ll} \because & \left\{ \begin{array}{ll} \bar{y}=\frac{\sum^{n}_{i=1}y_{i}}{n} \\ \sigma={\rm stand\ error\ of\ }y \end{array} \right.\\ \therefore & Var(\bar{y})=\frac{1}{n^{2}}\cdot\sum_{i=1}^{n}var(y_{i})=\frac{1}{n^{2}}\cdot{n\cdot\sigma^{2}}\\ \therefore & Var(\bar{y})=\frac{\sigma^{2}}{n} \end{array}$

3. $SE(\hat{\beta}_{1})=Var(\hat{\beta}_{1})=\frac{\sigma^{2}}{S_{xx}}$

   • $E(\hat{\beta}_{1})=\beta_{1}$

   $\begin{array}{ll} \because & \hat{\beta}_{1}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-n\cdot\bar{x}\bar{y}}{\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^{2}}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-\bar{x}\sum_{i=1}^n{y_{i}}}{\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^{2}}\\ & =\sum_{i=1}^{n}y_i\bigg(\frac{x_{i}-\bar{x}}{\sum_{i=1}^{n}(x_{i}-\bar{x})^2}\bigg) =\sum_{i=1}^{n}y_{i}\cdot\bigg(\frac{x_{i}-\bar{x}}{S_{xx}}\bigg)\\ \because & {\rm set}\ c_{i}=\frac{x_{i}-\bar{x}}{S_{xx}}\\ \therefore & \hat{\beta}_1=\sum_{i=1}^{n}c_{i}\cdot y_{i}\\ \therefore & E(\hat{\beta}_{1})=E\bigg(\sum_{i=1}^{n}{c_{i}y_{i}}\bigg)=\sum_{i=1}^{n}{E(c_{i}y_{i})}=\sum_{i=1}^{n}{c_{i}E({y_{i})}}\\ & =\sum_{i=1}^{n}{c_{i}(\beta_{0}+\beta_{1}x_{i}+\epsilon)}=\beta_{0}\sum_{i=1}^{n}c_{i}+\beta_{1}\sum_{i=1}^{n}c_{i}x_{i}\\ \because & \sum_{i=1}^{n}c_{i}=\sum_{i=1}^{n}\frac{x_{i}-\bar{x}}{S_{xx}}=0\ {\rm and} \\ & \sum_{i=1}^{n}c_{i}x_{i}=\sum_{i=1}^{n}\frac{x_{i}^{2}-x_i\bar{x}}{S_{xx}}=\frac{\sum_{i=1}^{n}x_{i}^{2}-n\cdot \bar{x}^{2}}{S_{xx}}=\frac{S_{xx}}{S_{xx}}=1\\ \therefore & E(\hat{\beta}_{1})=\beta_{0}\cdot 0 +\beta_{1}\cdot 1=\beta_{1} \end{array}$

   • $E(\hat{\beta}_{0})=\beta_{0}$

   $\begin{array}{ll} \because & \hat{\beta}_{0}=\bar{y}-\hat{\beta}_{1}\bar{x}\\ \therefore & E(\hat{\beta}_{0})=E(\bar{y}-\hat{\beta}_{1}\bar{x})=\bar{y}-E(\hat{\beta}_{1})\cdot{\bar{x}}\\ \because & E(\hat{\beta}_{1})=\beta_{1}\\ \therefore & E(\hat{\beta}_{0})=\bar{y}-\beta_{1}\cdot{\bar{x}}\\ \because & \bar{y}=\beta_{0}+\beta_{1}x_{1}+\epsilon\ {\rm and}\ E(\epsilon)=0\\ \therefore & E(\hat{\beta}_{0})=\beta_{0} \end{array}$

   $\begin{array}{ll} \because & \hat{\beta}_{1}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-n\cdot\bar{x}\bar{y}}{\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^{2}}=\frac{\sum_{i=1}^{n}x_{i}y_{i}-\bar{x}\sum_{i=1}^n{y_{i}}}{\sum_{i=1}^{n}x_{i}^{2}-n\cdot\bar{x}^{2}}\\ & =\sum_{i=1}^{n}y_i\bigg(\frac{x_{i}-\bar{x}}{\sum_{i=1}^{n}(x_{i}-\bar{x})^2}\bigg) =\sum_{i=1}^{n}y_{i}\cdot\bigg(\frac{x_{i}-\bar{x}}{S_{xx}}\bigg)\\ \because & {\rm set}\ c_{i}=\frac{x_{i}-\bar{x}}{S_{xx}}\\ \therefore & \hat{\beta}_1=\sum_{i=1}^{n}c_{i}\cdot y_{i}\\ \therefore & Var(\hat{\beta}_{1})=Var\bigg(\sum_{i=1}^{n}{c_{i}y_{i}}\bigg)=\sum_{i=1}^{n}{Var(c_{i}y_{i})}=\sum_{i=1}^{n}{c_{i}^{2}Var({y_{i})}}\\ \because & {\rm assume}\ Var(y_{i})=\sigma^{2}\\ \therefore & Var(\hat{\beta}_1)= \sigma^{2}\cdot\sum_{i=1}^{n}\bigg(\frac{(x_{i}-\bar{x})^{2}}{S_{xx}^2}\bigg)=\sigma^{2}\cdot\frac{\sum_{i=1}^{n}(x_{i}-\bar{x})^2}{S_{xx}^{2}}\\ & =\sigma^{2}\cdot\frac{S_{xx}}{S_{xx}^{2}}=\frac{\sigma^{2}}{S_{xx}} \end{array}$

4. $SE(\hat{\beta}_{0})=Var(\hat{\beta}_{0})=\sigma^{2}\bigg[\frac{1}{n}+\frac{\bar{x}^{2}}{\sum_{i=1}^{n}(x_{i}-\bar{x})^{2}}\bigg]=\sigma^{2}\bigg[\frac{1}{n}+\frac{\bar{x}^{2}}{S_{xx}}\bigg]$

   $\begin{array}{ll} \because & \hat{\beta}_{0}=\bar{y}-\hat{\beta}_{1}\bar{x}\\ \therefore & {\rm SE}(\hat{\beta}_{0})=Var(\hat{\beta}_{0})=Var(\bar{y}-\hat{\beta}_{1}\bar{x})=Var(\bar{y})-Var(\hat{\beta}_{1})\bar{x}^2\\ & =\frac{\sigma^{2}}{n}+\frac{\sigma^{2}}{S_{xx}}\cdot{\bar{x}^2}=\sigma^{2}\bigg[\frac{1}{n}+\frac{\bar{x}^{2}}{S_{xx}}\bigg]=\sigma^{2}\bigg[\frac{1}{n}+\frac{\bar{x}^{2}}{\sum_{i=1}^{n}(x_{i}-\bar{x})^{2}}\bigg] \end{array}$

5. $R^{2}=r^{2}$

   $\begin{array}{ll} \because & RSS = \sum_{i=1}^{n}{(y_{i}-\hat{y}_{i})^2} =\sum_{i=1}^{n}(y_{i}-\hat{\beta}_0-\hat{\beta}_{1}x_{i})^2 =\sum_{i=1}^{n}(y_{i}-\bar{y}+\hat{\beta}_{1}\bar{x}-\hat{\beta}_{1}x_{i})^{2} =\sum_{i=1}^{n}\big((y_{i}-\bar{y})^{2}-2\hat{\beta}_{1}(x_{i}-\bar{x})(y_{i}-\bar{y})+\hat{\beta}^{2}_{1}(x_{i}-\bar{x})^2\big)\\ & =\sum_{i=1}^{n}(y_{i}-\bar{y})^{2}-2\hat{\beta}_{1}\sum_{i=1}^{n}{(x_{i}-\bar{x})(y_{i}-\bar{y})}+\hat{\beta}_{1}^{2}\sum_{i=1}^{n}(\frac{y_{i}-\hat{\beta}_{0}}{\hat{\beta}_{1}}-\frac{\bar{y}-\hat{\beta}_{0}}{\hat{\beta}_{1}})(x_{i}-\bar{x})\\ & =\sum_{i=1}^{n}(y_{i}-\bar{y})^{2}-2\hat{\beta}_{1}\sum_{i=1}^{n}{(x_{i}-\bar{x})(y_{i}-\bar{y})}+\hat{\beta}_{1}^{2}\sum_{i=1}^{n}(\frac{y_{i}-\bar{y}}{\hat{\beta}_{1}})(x_{i}-\bar{x})\\ & =\sum_{i=1}^{n}(y_{i}-\bar{y})^{2}-2\hat{\beta}_{1}\sum_{i=1}^{n}{(x_{i}-\bar{x})(y_{i}-\bar{y})}+\hat{\beta}_{1}\sum_{i=1}^{n}(y_{i}-\bar{y})(x_{i}-\bar{x})\\ & =\sum_{i=1}^{n}(y_{i}-\bar{y})^{2}-\hat{\beta}_{1}\sum_{i=1}^{n}{(x_{i}-\bar{x})(y_{i}-\bar{y})}\\ & =S_{yy}-\hat{\beta}_{1}S_{xy}\\ \because & r = Cor(X,Y)=\frac{\sum_{i=1}^{n}(x_{i}-\bar{x})(y_{i}-\bar{y})}{\sqrt{\sum_{i=1}^{n}{(x_{i}-\bar{x})^{2}}}\sqrt{\sum_{i=1}^{n}{(y_{i}-\bar{y})^{2}}}} =\frac{S_{xy}}{\sqrt{S_{yy}}\sqrt{S_{xx}}}\\ \therefore & R^{2}=1-\frac{RSS}{TSS}=1-\frac{S_{yy}-\hat{\beta}_{1}S_{xy}}{S_{yy}}\\ & =\frac{\hat{\beta}_{1}S_{xy}}{S_{yy}}=\frac{\frac{S_{xy}}{S_{xx}}S_{xy}}{S_{yy}}=\frac{S_{xy}^{2}}{S_{yy}S_{xx}}\\ & = Cor^{2}(x,y)=r^{2} \end{array}$