Skip to content

Commit e2f3cb2

Browse files
JoramSochJoramSoch
authored
Merge pull request #324 from JoramSoch/master
added proof "mvn-jmc"
2 parents 40da217 + beecd8d commit e2f3cb2

2 files changed

Lines changed: 272 additions & 0 deletions

File tree

‎I/LoC.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,7 @@ Templates: **[Proof](/P/-temp-)** – *[Definition](/D/-temp-)*
554554
- **[Linear transformation](/P/mvn-ltt)**
555555
- **[Marginal distributions](/P/mvn-marg)**
556556
- **[Conditional distributions](/P/mvn-cond)**
557+
- **[Joint distribution](/P/mvn-jmc)**
557558
- **[Conditions for independence](/P/mvn-ind)**
558559
- **[Independence of products](/P/mvn-indprod)**
559560
- **[Drawing samples](/P/mvn-samp)**

‎P/mvn-jmc.tex‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
\documentclass[a4paper,12pt]{article}
2+
3+
%%% Packages %%%
4+
\usepackage{fullpage}
5+
\usepackage{amsmath}
6+
\usepackage{amssymb}
7+
\usepackage{gensymb}
8+
\usepackage{url}
9+
\usepackage{csquotes}
10+
\usepackage{enumitem}
11+
\usepackage{setspace}
12+
\usepackage[utf8]{inputenc}
13+
\usepackage[bottom,hang,flushmargin]{footmisc}
14+
15+
%%% Settings %%%
16+
\setlength{\parindent}{0pt}
17+
\frenchspacing
18+
\urlstyle{same}
19+
\MakeOuterQuote{"}
20+
\setlist{nolistsep}
21+
\setlist[itemize]{leftmargin=*}
22+
\setlist[enumerate]{leftmargin=*}
23+
24+
25+
\begin{document}
26+
27+
28+
**Theorem:** Let $X = [X_1,\ldots,X_m]^\mathrm{T}$ be an $m$-dimensional [multivariate normal](/D/mvn) [random vector](/D/rvec)
29+
30+
\begin{equation} \label{eq:mvn-marg}
31+
X \sim N(\mu_X, \Sigma_{XX})
32+
\quad \mbox{with} \quad
33+
\mu_X \in \mathbb{R}^m
34+
\quad \mbox{and} \quad
35+
\Sigma_{XX} \in \mathbb{R}^{m \times m}
36+
\end{equation}
37+
38+
and let $Y = [Y_1,\ldots,Y_n]^\mathrm{T}$ be an $n$-dimensional [conditionally](/D/dist-cond) [multivariate normal distributed](/D/mvn)
39+
40+
\begin{equation} \label{eq:mvn-cond}
41+
Y|X \sim N(AX+b, \Sigma_{YY})
42+
\quad \mbox{with} \quad
43+
\Sigma_{YY} \in \mathbb{R}^{n \times n}
44+
\quad \mbox{as well as} \quad
45+
A \in \mathbb{R}^{n \times m}
46+
\quad \mbox{and} \quad
47+
b \in \mathbb{R}^n \; .
48+
\end{equation}
49+
50+
Then, the $(m+n)$-dimensional random vector $Z = \begin{pmatrix} X \\ Y \end{pmatrix}$ is [jointly](/D/dist-joint) [multivariate normal distributed](/D/mvn)
51+
52+
\begin{equation} \label{eq:mvn-joint}
53+
Z \sim N(\mu_{X,Y}, \Sigma_{X,Y})
54+
\end{equation}
55+
56+
with the [multivariate mean](/D/mvn)
57+
58+
\begin{equation} \label{eq:mvn-joint-mean}
59+
\mu_{X,Y} = \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \in \mathbb{R}^{m+n}
60+
\end{equation}
61+
62+
and the [covariance matrix](/D/mvn)
63+
64+
\begin{equation} \label{eq:mvn-joint-cov}
65+
\Sigma_{X,Y} =
66+
\begin{pmatrix}
67+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
68+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
69+
\end{pmatrix} \in \mathbb{R}^{(m+n) \times (m+n)} \; .
70+
\end{equation}
71+
72+
73+
**Proof:** The [probability density function for the maginal distribution](/P/mvn-pdf) of $X$ is
74+
75+
\begin{equation} \label{eq:mvn-marg-pdf}
76+
p(x) = \frac{1}{\sqrt{(2\pi)^m |\Sigma_{XX}|}} \exp\left(-\frac{1}{2} (x-\mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x-\mu_X)\right) \; .
77+
\end{equation}
78+
79+
The [probability density function for the conditional distribution](/P/mvn-pdf) of $Y$ given $X$ is
80+
81+
\begin{equation} \label{eq:mvn-cond-pdf}
82+
p(y|x) = \frac{1}{\sqrt{(2\pi)^n |\Sigma_{YY}|}} \exp\left(-\frac{1}{2} (y-(Ax+b))^\mathrm{T} \Sigma_{YY}^{-1} (y-(Ax+b))\right) \; .
83+
\end{equation}
84+
85+
According to the [law of conditional probability](/D/prob-cond), we have
86+
87+
\begin{equation} \label{eq:prob-cond}
88+
p(y|x) = \frac{p(x,y)}{p(x)} \; ,
89+
\end{equation}
90+
91+
such that
92+
93+
\begin{equation} \label{eq:prob-joint}
94+
p(x,y) = p(y|x) \cdot p(x) \; .
95+
\end{equation}
96+
97+
With \eqref{eq:mvn-marg-pdf} and \eqref{eq:mvn-cond-pdf}, we get:
98+
99+
\begin{equation} \label{eq:mvn-joint-pdf-s1}
100+
\begin{split}
101+
p(x,y)
102+
&= \frac{1}{\sqrt{(2\pi)^n |\Sigma_{YY}|}} \exp\left(-\frac{1}{2} (y-(Ax+b))^\mathrm{T} \Sigma_{YY}^{-1} (y-(Ax+b))\right) \cdot \\
103+
&\hphantom{=}\; \frac{1}{\sqrt{(2\pi)^m |\Sigma_{XX}|}} \exp\left(-\frac{1}{2} (x-\mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x-\mu_X)\right) \\
104+
&= \frac{1}{\sqrt{(2\pi)^{m+n} |\Sigma_{XX}| |\Sigma_{YY}|}} \exp\left(-\frac{1}{2} \left[ (x-\mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x-\mu_X) + (y-(Ax+b))^\mathrm{T} \Sigma_{YY}^{-1} (y-(Ax+b)) \right] \right) \; .
105+
\end{split}
106+
\end{equation}
107+
108+
The determinant of a block matrix is:
109+
110+
\begin{equation} \label{eq:block-det}
111+
\left| \begin{pmatrix} A & B \\ C & D \end{pmatrix} \right|
112+
= |A| \cdot | D - C A^{-1} B | \; .
113+
\end{equation}
114+
115+
Thus, we get
116+
117+
\begin{equation} \label{eq:Sigma-yy-det}
118+
\begin{split}
119+
\left| \begin{pmatrix}
120+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
121+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
122+
\end{pmatrix} \right|
123+
&= |\Sigma_{XX}| \cdot | \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T} - A \Sigma_{XX} \Sigma_{XX}^{-1} \Sigma_{XX} A^\mathrm{T} | \\
124+
&= |\Sigma_{XX}| \cdot | \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T} - A \Sigma_{XX} A^\mathrm{T} | \\
125+
&= |\Sigma_{XX}| \cdot | \Sigma_{YY} + 0_{nn} | \\
126+
&= |\Sigma_{XX}| \cdot |\Sigma_{YY}| \; ,
127+
\end{split}
128+
\end{equation}
129+
130+
such that
131+
132+
\begin{equation} \label{eq:mvn-joint-pdf-s2}
133+
|\Sigma_{XX}| |\Sigma_{YY}| = |\Sigma_{X,Y}| \; .
134+
\end{equation}
135+
136+
The inverse of a block matrix is:
137+
138+
\begin{equation} \label{eq:block-inv}
139+
\begin{pmatrix} A & B \\ C & D \end{pmatrix}^{-1}
140+
= \begin{pmatrix}
141+
A^{-1} + A^{-1}B (D - CA^{-1}B)^{-1} CA^{-1} & -A^{-1}B (D - CA^{-1}B)^{-1} \\
142+
-(D - CA^{-1}B)^{-1} CA^{-1} & (D - CA^{-1}B)^{-1}
143+
\end{pmatrix} \; .
144+
\end{equation}
145+
146+
With
147+
148+
\begin{equation} \label{eq:Sigma-yy-inv-D}
149+
D - CA^{-1}B
150+
= \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T} - A \Sigma_{XX} \Sigma_{XX}^{-1} \Sigma_{XX} A^\mathrm{T}
151+
= \Sigma_{YY} \; ,
152+
\end{equation}
153+
154+
we obtain
155+
156+
\begin{equation} \label{eq:Sigma-yy-inv}
157+
\begin{split}
158+
\begin{pmatrix}
159+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
160+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
161+
\end{pmatrix}^{-1}
162+
&=\begin{pmatrix}
163+
\Sigma_{XX}^{-1} + \Sigma_{XX}^{-1} \Sigma_{XX} A^\mathrm{T} \Sigma_{YY}^{-1} A \Sigma_{XX} \Sigma_{XX}^{-1}
164+
& -\Sigma_{XX}^{-1} \Sigma_{XX} A^\mathrm{T} \Sigma_{YY}^{-1} \\
165+
-\Sigma_{YY}^{-1} A \Sigma_{XX} \Sigma_{XX}^{-1}
166+
& \Sigma_{YY}^{-1}
167+
\end{pmatrix} \\
168+
&=\begin{pmatrix}
169+
\Sigma_{XX}^{-1} + A^\mathrm{T} \Sigma_{YY}^{-1} A
170+
& -A^\mathrm{T} \Sigma_{YY}^{-1} \\
171+
-\Sigma_{YY}^{-1} A
172+
& \Sigma_{YY}^{-1}
173+
\end{pmatrix} \; ,
174+
\end{split}
175+
\end{equation}
176+
177+
such that
178+
179+
\begin{equation} \label{eq:mvn-joint-pdf-s3a}
180+
\begin{split}
181+
&\hphantom{=}\;
182+
\left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right)^\mathrm{T}
183+
\begin{pmatrix}
184+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
185+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
186+
\end{pmatrix}^{-1}
187+
\left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right) \\
188+
&=
189+
\begin{pmatrix} x - \mu_X \\ y - (A\mu_X + b) \end{pmatrix}^\mathrm{T}
190+
\begin{pmatrix}
191+
\Sigma_{XX}^{-1} + A^\mathrm{T} \Sigma_{YY}^{-1} A
192+
& -A^\mathrm{T} \Sigma_{YY}^{-1} \\
193+
-\Sigma_{YY}^{-1} A
194+
& \Sigma_{YY}^{-1}
195+
\end{pmatrix}
196+
\begin{pmatrix} x - \mu_X \\ y - (A\mu_X + b) \end{pmatrix} \\
197+
&=
198+
\begin{pmatrix}
199+
(x - \mu_X)^\mathrm{T} (\Sigma_{XX}^{-1} + A^\mathrm{T} \Sigma_{YY}^{-1} A)
200+
- (y - (A\mu_X + b))^\mathrm{T} \Sigma_{YY}^{-1} A \\
201+
- (x - \mu_X)^\mathrm{T} A^\mathrm{T} \Sigma_{YY}^{-1}
202+
+ (y - (A\mu_X + b))^\mathrm{T} \Sigma_{XX}^{-1}
203+
\end{pmatrix}^\mathrm{T}
204+
\begin{pmatrix} x - \mu_X \\ y - A\mu_X + b \end{pmatrix} \\
205+
&= (x - \mu_X)^\mathrm{T} (\Sigma_{XX}^{-1} + A^\mathrm{T} \Sigma_{YY}^{-1} A) (x - \mu_X) - (y - (A\mu_X + b))^\mathrm{T} \Sigma_{YY}^{-1} A (x - \mu_X) \\
206+
&- (x - \mu_X)^\mathrm{T} A^\mathrm{T} \Sigma_{YY}^{-1} (y - (A\mu_X + b)) + (y - (A\mu_X + b))^\mathrm{T} \Sigma_{XX}^{-1} (y - (A\mu_X + b)) \; .
207+
\end{split}
208+
\end{equation}
209+
210+
With
211+
212+
\begin{equation} \label{eq:Sigma-yy-inv-equiv}
213+
\begin{split}
214+
&\hphantom{=}\; \left[ (x - \mu_X)^\mathrm{T} A^\mathrm{T} \Sigma_{YY}^{-1} (y - (A\mu_X + b)) \right]^\mathrm{T} \\
215+
&= \left[ (y - (A\mu_X + b))^\mathrm{T} \Sigma_{YY}^{-1} A (x - \mu_X) \right] \in \mathbb{R}^{1 \times 1} \; ,
216+
\end{split}
217+
\end{equation}
218+
219+
we thus obtain
220+
221+
\begin{equation} \label{eq:mvn-joint-pdf-s3b}
222+
\begin{split}
223+
&\hphantom{=}\;
224+
\left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right)^\mathrm{T}
225+
\begin{pmatrix}
226+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
227+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
228+
\end{pmatrix}^{-1}
229+
\left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right) \\
230+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) + (x - \mu_X)^\mathrm{T} A^\mathrm{T} \Sigma_{YY}^{-1} A (x - \mu_X) \\
231+
&- 2 (y - (A\mu_X + b))^\mathrm{T} \Sigma_{YY}^{-1} A (x - \mu_X) + (y - (A\mu_X + b))^\mathrm{T} \Sigma_{XX}^{-1} (y - (A\mu_X + b)) \\
232+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) + (Ax - A\mu_X)^\mathrm{T} \Sigma_{YY}^{-1} (Ax - A\mu_X) \\
233+
&- 2 (y - (A\mu_X + b))^\mathrm{T} \Sigma_{YY}^{-1} A (x - \mu_X) + (y - (A\mu_X + b))^\mathrm{T} \Sigma_{XX}^{-1} (y - (A\mu_X + b)) \\
234+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) \\
235+
&+ \left[ Ax - A\mu_X - 2(y - (A\mu_X + b) + (y - (A\mu_X + b) \right]^\mathrm{T} \Sigma_{XX}^{-1} \left[ Ax - A\mu_X - 2(y - (A\mu_X + b) + (y - (A\mu_X + b) \right] \\
236+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) \\
237+
&+ \left[ Ax - A\mu_X - (y - (A\mu_X + b) \right]^\mathrm{T} \Sigma_{XX}^{-1} \left[ Ax - A\mu_X - (y - (A\mu_X + b) \right] \\
238+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) \\
239+
&+ \left[ Ax - A\mu_X - (y - (A\mu_X + b) \right]^\mathrm{T} \Sigma_{XX}^{-1} \left[ Ax - A\mu_X - (y - (A\mu_X + b) \right] \\
240+
&= (x - \mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x - \mu_X) + (y - (Ax + b))^\mathrm{T} \Sigma_{XX}^{-1} (y - (Ax + b)) \; ,
241+
\end{split}
242+
\end{equation}
243+
244+
such that
245+
246+
\begin{equation} \label{eq:mvn-joint-pdf-s3c}
247+
\begin{split}
248+
&\hphantom{=}\; (x-\mu_X)^\mathrm{T} \Sigma_{XX}^{-1} (x-\mu_X) + (y-(Ax+b))^\mathrm{T} \Sigma_{YY}^{-1} (y-(Ax+b)) \\
249+
&= \left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right)^\mathrm{T}
250+
\begin{pmatrix}
251+
\Sigma_{XX} & \Sigma_{XX} A^\mathrm{T} \\
252+
A \Sigma_{XX} & \Sigma_{YY} + A \Sigma_{XX} A^\mathrm{T}
253+
\end{pmatrix}^{-1}
254+
\left( \begin{pmatrix} x \\ y \end{pmatrix} - \begin{pmatrix} \mu_X \\ A\mu_X + b \end{pmatrix} \right) \\
255+
&= (z-\mu_{X,Y})^\mathrm{T} \Sigma_{X,Y}^{-1} (z-\mu_{X,Y}) \; .
256+
\end{split}
257+
\end{equation}
258+
259+
Plugging \eqref{eq:mvn-joint-pdf-s2} and \eqref{eq:mvn-joint-pdf-s3c} into \eqref{eq:mvn-joint-pdf-s1}, we finally get
260+
261+
\begin{equation} \label{eq:mvn-joint-pdf-s4}
262+
\begin{split}
263+
p(z)
264+
&= \frac{1}{\sqrt{(2\pi)^{m+n} |\Sigma_{X,Y}|}} \exp\left(-\frac{1}{2} (z-\mu_{X,Y})^\mathrm{T} \Sigma_{X,Y}^{-1} (z-\mu_{X,Y}) \right)
265+
\end{split}
266+
\end{equation}
267+
268+
which is the [probability density function of a multivariate normal distribution](/P/mvn-pdf) for the random vector $Z \in \mathbb{R}^{m+n}$ with multivariate mean $\mu_{X,Y}$ from \eqref{eq:mvn-joint-mean} and covariance matrix $\Sigma_{X,Y}$ from \eqref{eq:mvn-joint-cov}.
269+
270+
271+
\end{document}

0 commit comments

Comments
 (0)