Appendix03 Matrix Derivative

Types of matrix derivative

Derivatives with vectors

Vector-by-scalar

$$\begin{array}{c} \left\{ \begin{array}{ll} \mathbf{y} = \left [ \begin{matrix} y_{1} & y_{2} & \cdots & y_{m} \end{matrix} \right]^{T}\\ x \text{ is a scalar} \end{array} \right.\\\\ \big\Downarrow\\\\ \frac{\partial{\mathbf{y}}}{\partial{x}} = \left[ \begin{matrix} \frac{\partial{y_{1}}}{\partial{x}}\\ \frac{\partial{y_{2}}}{\partial{x}}\\ \vdots\\ \frac{\partial{y_{m}}}{\partial{x}}\\ \end{matrix} \right] \end{array}$$

Scalar-by-vector

$$\begin{array}{c} \left\{ \begin{array}{ll} y \text{ is a scalar}\\ \mathbf{x} = \left [ \begin{matrix} x_{1} & x_{2} & \cdots & x_{n} \end{matrix} \right] \end{array} \right.\\\\ \big\Downarrow\\\\ \frac{\partial{y}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{y}}{\partial{x_{1}}} & \frac{\partial{y}}{\partial{x_{2}}} & \cdots & \frac{\partial{y}}{\partial{x_{n}}} \end{matrix} \right] \end{array}$$

• application: $\nabla{f}$ $$\begin{array}{c} \left\{ \begin{array}{ll} f = f(x_{1}, x_{2}, \cdots, x_{n} )\\ \mathbf{x} = \left [ \begin{matrix} x_{1} & x_{2} & \cdots & x_{n} \end{matrix} \right] \end{array} \right.\\\\ \big\Downarrow\\\\ \nabla{f} = \left[ \begin{matrix} \frac{\partial{y}}{\partial{x_{1}}} \\ \frac{\partial{y}}{\partial{x_{2}}} \\ \vdots \\ \frac{\partial{y}}{\partial{x_{n}}} \end{matrix} \right] = \bigg(\frac{\partial{f}}{\partial{\mathbf{x}}}\bigg)^{T} \end{array}$$

Vector-by-vector

$$\begin{array}{c} \left\{ \begin{array}{ll} \mathbf{y} = \left[ \begin{matrix} y_{1} & y_{2} & \cdots & y_{m} \end{matrix} \right]^{T}\\ \mathbf{x} = \left[ \begin{matrix} x_{1} & x_{2} & \cdots & x_{n} \end{matrix} \right]^{T} \end{array} \right.\\\\ \big\Downarrow\\\\ \frac{\partial{\mathbf{y}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{y_{1}}}{\partial{x_{1}}} & \frac{\partial{y_{1}}}{\partial{x_{2}}} & \cdots & \frac{\partial{y_{1}}}{\partial{x_{n}}}\\ \frac{\partial{y_{2}}}{\partial{x_{1}}} & \frac{\partial{y_{2}}}{\partial{x_{2}}} & \cdots & \frac{\partial{y_{2}}}{\partial{x_{n}}}\\ \vdots & \vdots & \ddots & \vdots \\ \frac{\partial{y_{m}}}{\partial{x_{1}}} & \frac{\partial{y_{m}}}{\partial{x_{2}}} & \cdots & \frac{\partial{y_{m}}}{\partial{x_{n}}} \end{matrix} \right] \end{array}$$

Layout conventions

In the case of Vector-by-Vector $$\left\{ \begin{array}{ll} \mathbf{y} = \left[ \begin{matrix} y_{1} & y_{2} & \cdots & y_{m} \end{matrix} \right]^{T}\\ \mathbf{x} = \left[ \begin{matrix} x_{1} & x_{2} & \cdots & x_{n} \end{matrix} \right] \end{array} \right.$$

Numerator layout

• layout according to $\mathbf{y}$ and $\mathbf{x}^{T}$
• output size $m\times n$
• $\frac{\partial{\mathbf{y}}}{\partial{\mathbf{x}}}$
  • choose $\frac{\partial{y}}{\partial{\mathbf{x}}}$ as a row Vector.
  • choose $\frac{\partial{\mathbf{y}}}{\partial{x}}$ as a column Vector.

Denominator layout

• layout according to $\mathbf{y}^{T}$ and $\mathbf{x}$
• output size $n\times m$
• $\frac{\partial{\mathbf{y}}}{\partial{\mathbf{x}}}$
  • choose $\frac{\partial{y}}{\partial{\mathbf{x}}}$ as a column Vector.
  • choose $\frac{\partial{\mathbf{y}}}{\partial{x}}$ as a row Vector.

Some Proofs

$$\frac{\partial{\mathbf{Ax}}}{\partial{\mathbf{x}}}$$ $\mathbf{A}$

$$\left[ \begin{matrix} a_{11} & a_{12} & \cdots & a_{1n}\\ a_{21} & a_{22} & \cdots & a_{2n}\\ \vdots & \vdots & \ddots & \vdots \\ a_{m1} & a_{m2} & \cdots & a_{mn} \end{matrix} \right]$$

$\mathbf{x}$

$$\left[ \begin{matrix} x_{1}\\ x_{2}\\ \vdots\\ x_{n} \end{matrix} \right]$$

$\mathbf{Ax}$

$$\left[ \begin{matrix} \sum_{j=1}^{n}a_{1j}x_{j}\\ \sum_{j=1}^{n}a_{2j}x_{j}\\ \vdots\\ \sum_{j=1}^{n}a_{mj}x_{j}\\ \end{matrix} \right] = \left[ \begin{matrix} f_{1}(x_{1},x_{2},\cdots,x_{n})\\ f_{2}(x_{1},x_{2},\cdots,x_{n})\\ \vdots\\ f_{m}(x_{1},x_{2},\cdots,x_{n})\\ \end{matrix} \right]$$

Numerator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{Ax}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \vdots\\ \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \end{matrix} \right] = \left[ \begin{matrix} \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \vdots & \vdots & \ddots & \vdots\\ \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \end{matrix} \right] \\ = \left[ \begin{matrix} a_{11} & a_{12} & \cdots & a_{1n}\\ a_{21} & a_{22} & \cdots & a_{2n}\\ \vdots & \vdots & \ddots & \vdots \\ a_{m1} & a_{m2} & \cdots & a_{mn} \end{matrix} \right] = \mathbf{A} \end{array}$

Denominator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{Ax}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} & \cdots & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} \end{matrix} \right] \\ = \left[ \begin{matrix} \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \cdots & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}}\\ \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}}\\ \vdots & \vdots & \ddots & \vdots\\ \frac{\partial{f_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}} & \frac{\partial{f_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}} & \cdots & \frac{\partial{f_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \end{matrix} \right] = \left[ \begin{matrix} a_{11} & a_{21} & \cdots & a_{m1}\\ a_{12} & a_{22} & \cdots & a_{m2}\\ \vdots & \vdots & \ddots & \vdots \\ a_{1n} & a_{2n} & \cdots & a_{mn} \end{matrix} \right] = \mathbf{A}^{T} \end{array}$

---

$$\frac{\partial{\mathbf{x}^{T}\mathbf{A}}}{\partial{\mathbf{x}}}$$

$\mathbf{A}$

$$\left[ \begin{matrix} a_{11} & a_{12} & \cdots & a_{1m}\\ a_{21} & a_{22} & \cdots & a_{2m}\\ \vdots & \vdots & \ddots & \vdots \\ a_{n1} & a_{n2} & \cdots & a_{nm}\\ \end{matrix} \right]$$

$\mathbf{x}^{T}$

$$\left[ \begin{matrix} x_{1} & x_{2} & \cdots & x_{n} \end{matrix} \right]$$

$\mathbf{x}^{T}\mathbf{A}$

$$\begin{array}{ll} & \left[ \begin{matrix} \sum_{i=1}^{n}{a_{i1}x_{i}} & \sum_{i=1}^{n}{a_{i2}x_{i}} & \cdots & \sum_{i=1}^{n}{a_{im}x_{i}} \end{matrix} \right]\\ = & \left[ \begin{matrix} g_{1}(x_{1},x_{2},\cdots,x_{n}) & g_{2}(x_{1},x_{2},\cdots,x_{n}) & \cdots & g_{m}(x_{1},x_{2},\cdots,x_{n}) \end{matrix} \right] \end{array}$$

Numerator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{x}^{T}\mathbf{A}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \vdots\\ \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}}\\ \end{matrix} \right] = \left[ \begin{matrix} \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \vdots & \vdots & \ddots & \vdots\\ \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \end{matrix} \right] \\ = \left[ \begin{matrix} a_{11} & a_{21} & \cdots & a_{n1}\\ a_{12} & a_{22} & \cdots & a_{n2}\\ \vdots & \vdots & \ddots & \vdots \\ a_{1m} & a_{2m} & \cdots & a_{nm} \end{matrix} \right] = \mathbf{A}^{T} \end{array}$

Denominator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{x}^{T}\mathbf{A}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} & \cdots & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} \end{matrix} \right] \\ = \left[ \begin{matrix} \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \cdots & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}}\\ \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}}\\ \vdots & \vdots & \ddots & \vdots\\ \frac{\partial{g_{1}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}} & \frac{\partial{g_{2}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{b}}} & \cdots & \frac{\partial{g_{m}(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}}\\ \end{matrix} \right] = \left[ \begin{matrix} a_{11} & a_{12} & \cdots & a_{1m}\\ a_{21} & a_{22} & \cdots & a_{2m}\\ \vdots & \vdots & \ddots & \vdots \\ a_{n1} & a_{n2} & \cdots & a_{nm} \end{matrix} \right] = \mathbf{A} \end{array}$

$$\frac{\partial{\mathbf{x}^{T}\mathbf{Ax}}}{\partial{\mathbf{x}}}$$ $\mathbf{A}$

$$\left[ \begin{matrix} a_{11} & a_{12} & \cdots & a_{1n}\\ a_{21} & a_{22} & \cdots & a_{2n}\\ \vdots & \vdots & \ddots & \vdots \\ a_{n1} & a_{n2} & \cdots & a_{nn} \end{matrix} \right]$$

$\mathbf{x}$

$$\left[ \begin{matrix} x_{1}\\ x_{2}\\ \vdots\\ x_{n} \end{matrix} \right]$$

$\mathbf{x}^{T}\mathbf{Ax}$

$$\begin{array}{ll} \left[ \begin{matrix} \sum_{i=1}^{n}a_{i1}x_{i} & \sum_{i=1}^{n}a_{i2}x_{i} & \cdots & \sum_{i=1}^{n}a_{in}x_{i} \end{matrix} \right] \cdot \left[ \begin{matrix} x_{1}\\ x_{2}\\ \vdots\\ x_{n} \end{matrix} \right]\\ = \sum_{j=1}^{n}\big(x_{j}\sum_{i=1}^{n}a_{ij}x_{i}\big) = F(x_{1},x_{2},\cdots,x_{n}) \end{array}$$

Numerator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{x}^{T}\mathbf{Ax}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} \end{matrix} \right] = \left[ \begin{matrix} \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} & \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} & \cdots & \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}} \end{matrix} \right]\\ = \left[ \begin{matrix} \sum_{i=1}^{n}a_{i1}x_{i} + \sum_{j=1}^{n}a_{1j}x_{j} & \sum_{i=1}^{n}a_{i2}x_{i} + \sum_{j=1}^{n}a_{2j}x_{j} & \cdots & \sum_{i=1}^{n}a_{in}x_{i} + \sum_{j=1}^{n}a_{nj}x_{j} \end{matrix} \right] \\ = \left[ \begin{matrix} \sum_{i=1}^{n}a_{i1}x_{i} & \sum_{i=1}^{n}a_{i2}x_{i} & \cdots & \sum_{i=1}^{n}a_{in}x_{i} \end{matrix} \right] + \left[ \begin{matrix} \sum_{j=1}^{n}a_{1j}x_{j} & \sum_{j=1}^{n}a_{2j}x_{j} & \cdots & \sum_{j=1}^{n}a_{nj}x_{j} \end{matrix} \right]\\ = \mathbf{x}^{T}\mathbf{A} + (\mathbf{Ax})^{T} = \mathbf{x}^{T}(\mathbf{A}+\mathbf{A}^{T}) \end{array}$

Denominator Layout

$\begin{array}{ll} \frac{\partial{\mathbf{x}^{T}\mathbf{Ax}}}{\partial{\mathbf{x}}} = \left[ \begin{matrix} \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{\mathbf{x}}} \end{matrix} \right] = \left[ \begin{matrix} \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{1}}} \\ \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{2}}} \\ \vdots \\ \frac{\partial{F(x_{1},x_{2},\cdots,x_{n})}}{\partial{x_{n}}} \end{matrix} \right] = \left[ \begin{matrix} \sum_{i=1}^{n}a_{i1}x_{i} + \sum_{j=1}^{n}a_{1j}x_{j} \\ \sum_{i=1}^{n}a_{i2}x_{i} + \sum_{j=1}^{n}a_{2j}x_{j} \\ \vdots \\ \sum_{i=1}^{n}a_{in}x_{i} + \sum_{j=1}^{n}a_{nj}x_{j} \end{matrix} \right] \\ = \left[ \begin{matrix} \sum_{i=1}^{n}a_{i1}x_{i} \\ \sum_{i=1}^{n}a_{i2}x_{i} \\ \vdots \\ \sum_{i=1}^{n}a_{in}x_{i} \end{matrix} \right] + \left[ \begin{matrix} \sum_{j=1}^{n}a_{1j}x_{j} \\ \sum_{j=1}^{n}a_{2j}x_{j} \\ \vdots \\ \sum_{j=1}^{n}a_{nj}x_{j} \end{matrix} \right]\\ = (\mathbf{x}^{T}\mathbf{A})^{T} + \mathbf{Ax} = (\mathbf{A}^{T} + \mathbf{A})\mathbf{x} \end{array}$