Appendix A
With the stochastic maximum principle (4)–(6) to LQG control system (3) involving multiple state delays and multiplicative noise, we can obtain for \(k=N\),
$$\begin{aligned} 0&=\sum \limits _{i=0}^d\big (\mathcal {D}'(N){\mathcal {P}}_{N+1}^0\mathcal {C}_i(N)+\gamma \bar{\mathcal {D}}'(N){\mathcal {P}}_{N+1}^0\bar{\mathcal {C}}_i(N)\big )x_{N-i}\\&\quad +\big (\mathcal {D}'(N){\mathcal {P}}_{N+1}^0D(N)+\gamma \bar{\mathcal {D}}'(N){\mathcal {P}}_{N+1}^0\bar{\mathcal {D}}(N)+R_N\big )\\&\quad \times u_N+\mathcal {D}'(N){\mathcal {P}}_{N+1}^0{\bar{\mu }}_N+\bar{\mathcal {D}}'(N){\mathcal {P}}_{N+1}^0\tau . \end{aligned}$$
Using Eqs. (8) and (9), the optimal controller \(u_N\) is as
$$\begin{aligned} u_N=&-\varOmega _N^{-1}\sum \limits _{i=0}^dN_N^ix_{N-i}-\varOmega _N^{-1}\varSigma _N, \end{aligned}$$
where \(\varSigma _N=\mathcal {D}'(N){\mathcal {P}}_{N+1}^0{\bar{\mu }}_N+\bar{\mathcal {D}}'(N){\mathcal {P}}_{N+1}^0\tau\). From (4) and (5), we also have
$$\begin{aligned} \zeta _{N-1}&=\mathrm{E}\Bigg [(\mathcal {C}_N^0)'(N){\mathcal {P}}_{N+1}^0x_{N+1}|{\mathcal {F}}_{N-1}\Bigg ]+Q_Nx_N\\&=\mathrm{E}\Bigg [\Bigg (\sum \limits _{i=0}^d(\mathcal {C}_N^0)'(N){\mathcal {P}}_{N+1}^0\mathcal {C}_N^i(N)-(N_N^0)'\varOmega _N^{-1}N_N^i\Bigg )\\&\quad \times x_{N-i}- (N_N^0)'\varOmega _N^{-1}\mathcal {D}_N'(N){\mathcal {P}}_{N+1}^0\mu _N + (\mathcal {C}_N^0)'(N)\\&\quad \times {\mathcal {P}}_{N+1}^0\mu _N|{\mathcal {F}}_{N-1}\Bigg ]+Q_Nx_N. \end{aligned}$$
Substituting (7), \(\zeta _{N-1}\) yields
$$\begin{aligned} \zeta _{N-1}&=\sum \limits _{i=1}^d{\mathcal {P}}_N^ix_{N-i}+{\mathcal {P}}_N^0x_N+(\mathcal {C}_0'(N){\mathcal {P}}_{N+1}^0{\bar{\mu }}_N\\&\quad +\bar{\mathcal {C}}_0'(N){\mathcal {P}}_{N+1}^0\tau -(N_N^0)'\varOmega _N^{-1}\mathcal {D}'(N){\mathcal {P}}_{N+1}^0\\&\quad \times {\bar{\mu }}_N-(N_N^0)'\varOmega _N^{-1}\bar{\mathcal {D}}'(N){\mathcal {P}}_{N+1}^0\tau )\\&=\sum \limits _{j=0}^d{\mathcal {P}}_{N}^jx_{N-j}+\varPhi _N, \end{aligned}$$
where \(\varPhi _N\) satisfied (12) with the terminal values being zero.
Now, we have verified (11) for \(k=N\). Supposing that \(\zeta _{k-1}\) are as (11) for all \(k\geqslant n+1\), we will show that (11) also holds for \(k=n\). For \(k=n+1\), with (3) and (11), \(\zeta _n\) can be calculated as
$$\begin{aligned} \zeta _n&=\sum \limits _{j=0}^d{\mathcal {P}}_{n+1}^jx_{n+1-j}+\varPhi _{n+1}\nonumber \\&=\sum \limits _{j=1}^d{\mathcal {P}}_{n+1}^jx_{n+1-j}+{\mathcal {P}}_{n+1}^0\sum \limits _{i=0}^d\big (\mathcal {C}_n^i(n)x_{n-i}+\mathcal {D}(n)u_{n}\nonumber \\&\quad +\mu _{n}\big )+\varPhi _{n+1} . \end{aligned}$$
(22)
Inserting \(\zeta _n\) to (6), (6) will become
$$\begin{aligned} 0&=\mathrm{E}\Bigg [\sum \limits _{j=0}^d\big (\mathcal {D}_n'(n){\mathcal {P}}_{n+1}^{j+1} + \mathcal {D}_n'(n){\mathcal {P}}_{n+1}^0\mathcal {C}_n^j(n)\big )x_{n-j} + \mathcal {D}_n'(n)\\&\quad \times \!{\mathcal {P}}_{n+1}^0\mathcal {D}_n(n)u_n \!+\! \mathcal {D}_n'(n){\mathcal {P}}_{n+1}^0\mu _n \!+\! \mathcal {D}_n'(n)\varPhi _{n+1}|{\mathcal {F}}_{n-1}\Bigg ] \!+\!R_nu_n\\&=\sum \limits _{j=0}^d N_{n}^jx_{n-j}+ \varOmega _nu_n + \mathcal {D}'(n)\left( {\mathcal {P}}_{n+1}^0{\bar{\mu }}_n + \varPhi _{n+1}\right) \\&\quad + \bar{\mathcal {D}}'(n){\mathcal {P}}_{n+1}^0\tau . \end{aligned}$$
Thus, the optimal controller is given by
$$\begin{aligned} u_n=-\varOmega _n^{-1}\sum \limits _{j=0}^dN_{n}^jx_{n-j}-\varOmega _n^{-1}\varPhi _n. \end{aligned}$$
(23)
In virtue of equations (3), (5) and (23), \(\zeta _{n-1}\) yields that
$$\begin{aligned}&\zeta _{n-1}\\&\quad =\mathrm{E}\Bigg [ \sum \limits _{m=0}^{d-1} (\mathcal {C}_{n+m}^m)'(n + m)\zeta _{n+m} + (\mathcal {C}_{n+d}^d)'(n + d)\\&\qquad \times\Big (\sum \limits _{j=1}^d{\mathcal {P}}_{n+d+1}^j x_{n+d+1-j}+{\mathcal {P}}_{n+d+1}^0\Big (\sum \limits _{i=0}^d\mathcal {C}_{n+d}^i(n + d)x_{n+d-i}\\&\qquad +\mu _{n+d}+\mathcal {D}_{n+d}(n + d)u_{n+d}\Big )+\varPhi _{n+d+1}\Big )|{\mathcal {F}}_{n-1}\Bigg ]+Q_nx_n\\&\quad =\mathrm{E}\Bigg [\sum \limits _{m=0}^{d-1}(\mathcal {C}_{n+m}^m)'(n + m)\zeta _{n+m} + \sum \limits _{j=0}^d\big ((\mathcal {C}_{n+d}^d)'(n + d){\mathcal {P}}_{n+d+1}^{j+1}\\&\qquad +(\mathcal {C}_{n+d}^d)'(n + d){\mathcal {P}}_{n+d+1}^0\mathcal {C}_{n+d}^j(n + d)-(N_{n+d}^d)'\varOmega _{n+d}^{-1}\\&\qquad \times N_{n+d}^j\big )x_{n+d-j}-(N_{n+d}^d)'\varOmega _{n+d}^{-1}\varSigma _{n+d}+(\mathcal {C}_{n+d}^d)'(n + d)\\&\qquad \times ({\mathcal {P}}_{n+d+1}^0\mu _{n+d}+\varPhi _{n+d+1})|{\mathcal {F}}_{n-1}\Bigg ]+Q_nx_n \\&\quad =\mathrm{E}\Bigg [\sum \limits _{m=0}^{d-2}(\mathcal {C}_{n+m}^m)'(n + m)\zeta _{n+m}+\sum \limits _{j=0}^d\big ((\mathcal {C}_{n+d-1}^{d-1})'(n + d - 1)\\&\qquad \times {\mathcal {P}}_{n+d}^{j+1}+(\mathcal {C}_{n+d-1}^{d-1})'(n + d - 1){\mathcal {P}}_{n+d}^0\mathcal {C}_{n+d-1}^j(n + d - 1)\\&\qquad +(\mathcal {C}_{n+d}^d)'(n + d){\mathcal {P}}_{n+d+1}^{j+2}+(\mathcal {C}_{n+d}^d)'(n + d){\mathcal {P}}_{n+d+1}^0\\&\qquad \times \mathcal {C}_{n+d}^{j+1}(n + d)-(N_{n+d}^d)'\varOmega _{n+d}N_{n+d}^{j+1}+({\mathcal {P}}_{n+d}^d)'\\&\qquad \times \mathcal {C}_{n+d-1}^j(n + d - 1)\big )x_{n+d-1-j}+(N_{n+d-1}^{d-1})'u_{n+d-1}\\&\qquad - (N_{n+d}^d)'\varOmega _{n+d}^{-1}\varSigma _{n+d} + (\mathcal {C}_{n+d}^d)'(n + d)({\mathcal {P}}_{n+d+1}^0\mu _{n+d} \\&\qquad + \varPhi _{n+d+1})+(\mathcal {C}_{n+d-1}^{d-1})'(n + d - 1)({\mathcal {P}}_{n+d}^0g_{n+d-1}+\varPhi _{n+d})\\&\qquad +({\mathcal {P}}_{n+d}^{d})'\mu _{n+d-1}|{\mathcal {F}}_{n-1}\Bigg ]+Q_nx_n\\&\quad =\mathrm{E}\Bigg [\sum \limits _{m=0}^{d-3}(\mathcal {C}_{n+m}^m)'(n + m)\zeta _{n+m} + \sum \limits _{j=0}^d\sum \limits _{i=d-2}^d \big ((\mathcal {C}_{n+i}^i)'(n + i)\\&\qquad \times {\mathcal {P}}_{n+i+1}^0\mathcal {C}_{n+i}^{i+j-d+2}(n + i)+(\mathcal {C}_{n+i}^i)'(n + i){\mathcal {P}}_{n+i+1}^{i+j-d+3}\\&\qquad +({\mathcal {P}}_{n+i+1}^{i+1})'\mathcal {C}_{n+i}^{j+1}(n+i)-(N_{n+i}^i)'\varOmega _{n+i}^{-1}N_{n+i}^{i+j-d+2}\big )\\&\qquad \times x_{n+d-2-j}-\sum \limits _{i=d-1}^d \big ((N_{n+i}^i)'\varOmega _{n+i}^{-1}\varSigma _{n+i}+(\mathcal {C}_{n+i}^i)'(n + i)\\&\qquad \times ({\mathcal {P}}_{n+i+1}^0\mu _{n+i}+\varPhi _{n+i+1})+({\mathcal {P}}_{n+i+1}^{i+1})'\mu _{n+i}\big )|{\mathcal {F}}_{n-1}\Bigg ]\\&\qquad +Q_nx_n. \end{aligned}$$
Plugging (3) and (23) into the above equation for times d, we can calculate \(\zeta _{n-1}\) as follows:
$$\begin{aligned}&\zeta _{n-1}\\&\quad =\mathrm{E}\Bigg [\sum \limits _{j=0}^d\sum \limits _{i=0}^d \Bigg ((\mathcal {C}_{n+i}^i)'(n + i){\mathcal {P}}_{n+i+1}^0\mathcal {C}_{n+i}^{i+j}(n + i) \\&\qquad + (\mathcal {C}_{n+i}^i)'(n + i) {\mathcal {P}}_{n+i+1}^{i+j}+({\mathcal {P}}_{n+i+1}^{i+1})'\mathcal {C}_{n+i}^{j+1}(n+i)\\&\qquad -(N_{n+i}^i)'\varOmega _{n+i}^{-1}N_{n+i}^{i+j}\Bigg ) x_{n+1-j}+\sum \limits _{i=0}^d\Bigg ((-N_{n+i}^i)'\varOmega _{n+i}^{-1}\varSigma _{n+i}\\&\qquad +(\mathcal {C}_{n+i}^i)'(n+i) ({\mathcal {P}}_{n+i+1}^0\mu _{n+i}+\varPhi _{n+i+1})\\&\qquad +({\mathcal {P}}_{n+i+1}^{i+1})'\mu _{n+i}\Bigg )|{\mathcal {F}}_{n-1}\Bigg ]+Q_nx_n, \end{aligned}$$
where \({\mathcal {P}}_{n+i+1}^{i+j}=0\) for \(i+j>d\) from Remark 1, and then
$$\begin{aligned}&\zeta _{n-1}\\&\quad =\sum \limits _{j=0}^d\sum \limits _{i=0}^{d-j}\big (\mathcal {C}_i'(n + i){\mathcal {P}}_{n+i+1}^0\mathcal {C}_{i+j}(n + i)\\&\qquad +\gamma \bar{\mathcal {C}}_i'(n + i){\mathcal {P}}_{n+i+1}^0 \bar{\mathcal {C}}_{i+j}(n + i)+\mathcal {C}_i'(n + i){\mathcal {P}}_{n+i+1}^{i+j+1}\\&\qquad +({\mathcal {P}}_{n+i+1}^{i+1})'\mathcal {C}_{i+j}(n + i)-(N_{n+i}^i)'\varOmega _{n+i}^{-1}N_{n+i}^{i+j}\big )x_{n-j}\\&\qquad +\sum \limits _{i=0}^d\big ((-N_{n+i}^i)'\varOmega _{n+i}^{-1}\varSigma _{n+i}+\mathcal {C}_i'(n + i)({\mathcal {P}}_{n+i+1}^0{\bar{\mu }}_{n+i}\\&\qquad + \varPhi _{n+i+1}) + \bar{\mathcal {C}}_i'(n + i)({\mathcal {P}}_{n+i+1}^0\tau + ({\mathcal {P}}_{n+i+1}^{i+1})'{\bar{\mu }}_{n+i}\big )+Q_nx_n. \end{aligned}$$
After inserting (7), we can summarize that
$$\begin{aligned} \zeta _{n-1}=\sum \limits _{j=0}^d{\mathcal {P}}_{n}^jx_{n-j}+\varPhi _{n}. \end{aligned}$$
This completes the proof of the lemma.
Appendix B
(Necessity) Suppose that there exists the unique \({\mathcal {F}}_{k-1}\)-measurable \(u_k\) to make the cost function (2) minimized. We will show that \(\varOmega _k, k=0,\dots ,N\) are positive definite by induction and the optimal controller can be designed as (13). Define
$$\begin{aligned} J(k)= \mathrm{E}\Bigg [\sum \limits _{i=k}^Nx_i'Q_ix_i+u_i'R_iu_i+x_{N+1}'{\mathcal {P}}_{N+1}^0x_{N+1}\Bigg ]. \end{aligned}$$
When \(k=N\), J(N) is presented as
$$\begin{aligned} J(N)&= u_N'\varOmega _Nu_N + 2u_N(\mathcal {D}(N){\mathcal {P}}_{N+1}^0{\bar{\mu }}_N + \bar{\mathcal {D}}(N){\mathcal {P}}_{N+1}^0\tau )\\&\quad + {\rm Tr}[{\mathcal {P}}_{N+1}^0Q_{\mu _N}], \end{aligned}$$
where \(x_N=0\) and \(x_{N-j}=0\) for \(j=0,\ldots ,d\) as the uniqueness of the optimal controller is unrelated with \(x_k\).
As J(N) can be expressed as a quadratic function of \(u_N\), and the performance index must be positive, it can be obviously know that \(\varOmega _N>0\), i.e., \(\varOmega _k\) is positive definite for \(k=N\). Assuming \(\varOmega _k>0\) for all \(k\geqslant n+1\), we will prove that \(\varOmega _n>0\). With (3), (5) and (6), for \(k\geqslant n+1\), we construct that
$$\begin{aligned}&\mathrm{E}\Bigg [x_k'\zeta _{k-1}-x_{k+1}'\zeta _k\Bigg ]\nonumber \\&\quad =\mathrm{E}\Bigg [x_k'\mathrm{E}[\sum \limits _{m=0}^d\mathcal {C}_{k+m}^m(k+m)\zeta _{k+m}|{\mathcal {F}}_{k-1}]+x_k'Q_kx_k\nonumber \\&\qquad -\Bigg(\sum \limits _{i=0}^d\mathcal {C}_k^i(k)x_{k-i}+\mathcal {D}_k(k)u_k+\mu _k\Bigg )'\zeta _k\Bigg ]\nonumber \\&\quad =\mathrm{E}\Bigg [x_k'Q_kx_k-u_k'\mathrm{E}[\mathcal {D}'(k)\zeta _k|{\mathcal {F}}_{k-1}]-\mu _k'\zeta _k\Bigg ]\nonumber \\&\quad =\mathrm{E}\Bigg [x_k'Q_kx_k+u_k'R_ku_k-\mu _k'\zeta _k\Bigg ]. \end{aligned}$$
(24)
To obtain the form of J(N), we add both sides of (24) from \(k=n+1\) to \(k=N\), we have
$$\begin{aligned} \mathrm{E}[x_{n+1}'\zeta _n-x_{N+1}'\zeta _N]= \sum \limits _{k=n+1}^N \mathrm{E}\big [x_k'Q_kx_k+u_k'R_ku_k-\mu _k'\zeta _k\big ]. \end{aligned}$$
Then,
$$\begin{aligned}&\mathrm{E}\Bigg [\sum \limits _{k=n+1}^N\Bigg (x_k'Q_kx_k+u_k'R_ku_k\Bigg )+x_{N+1}'{\mathcal {P}}_{N+1}^0x_{N+1}\Bigg ]\\&=\mathrm{E}\big [x_{n+1}'\zeta _n-\sum \limits _{k=n}^N\mu _k'\zeta _k\big ]. \end{aligned}$$
Compared with (2), it yields that
$$\begin{aligned} J(n) = \big [x_n'Q_nx_n+u_n'R_nu_n\big ]+\mathrm{E}\Big [x_{n+1}'\zeta _n + \sum \limits _{k=n+1}^{N} \mu _k'\zeta _k\Big ]. \end{aligned}$$
(25)
Setting \(x_n=0\) and \(x_{n-i}=0\), for \(i=0,\ldots ,d\) as the same as the condition \(k=N\), and plugging (11) into (25), we obtain
$$\begin{aligned} J(n)&=\mathrm{E}\Bigg [u_n'R_nu_n+u_n'D_n'(n)\zeta _n+\sum \limits _{k=n}^N\mu _k'\zeta _k\Bigg ]\\&= \mathrm{E}\big [u_n'R_nu_n+u_n'\mathcal {D}_n'(n)({\mathcal {P}}_{n+1}^0\mathcal {D}_n(n)u_n+\mu _n)\\&\quad +u_n'\mathcal {D}_n'(n)\varPhi _{n+1}+\sum \limits _{k=n}^N\mu _k'\zeta _k\big ]\\&=u_n'\varOmega _nu_n+u_n'\big (\mathcal {D}'(n)({\mathcal {P}}_{n+1}^0{\bar{\mu }}_n+\varPhi _{n+1})\\&\quad +\bar{\mathcal {D}}'(n){\mathcal {P}}_{n+1}^0\tau \big )+\sum \limits _{k=n+1}^N\mu _k'\zeta _k. \end{aligned}$$
Similarly to the case \(\varOmega _N>0\) above, we obviously get \(\varOmega _n>0\) for all \(k=0,\ldots , N\). This completes the proof of necessity.
(Sufficiency) Suppose that \(\varOmega _k>0\) for \(k=0,\dots ,N\) is ture, we will show the existence of the unique \({\mathcal {F}}_{k-1}\)-measurable \(u_k\) to minimize (2). Make the definition:
$$\begin{aligned}&V(x_k)\\&\quad =\mathrm{E}\Bigg [x_k'{\mathcal {P}}_k^0x_k+2x_k'\sum \limits _{j=1}^{d}{\mathcal {P}}_k^jx_{k-j}\\&\qquad +\sum \limits _{j=1}^{d}\sum \limits _{i=1}^{d}\sum \limits _{l=0}^{d-1}x_{k-j}' \Bigg [\mathcal {C}_{j+l}'(k + l) {\mathcal {P}}_{k+l+1}^0\mathcal {C}_{i+l}(k + l)\\&\qquad +\gamma \bar{\mathcal {C}}_{j+l}'(k + l){\mathcal {P}}_{k+l+1}^0 \bar{\mathcal {C}}_{i+l}(k + l)+\mathcal {C}_{j+l}'(k + l){\mathcal {P}}_{k+l+1}^{i+l+1}\\&\qquad +({\mathcal {P}}_{k+l+1}^{j+l+1})' \mathcal {C}_{i+l}(k + l)-(N_{k+l}^{j+l})'\varOmega _{k+l}^{-1}N_{k+l}^{i+l}\Bigg ]x_{k-i}\\&\qquad +2x_k'\varPhi _k\Bigg ]. \end{aligned}$$
First, as \(k=k+1\), using the equivalent substitution \(l=l+1\), \(j=j-1\), and \(i=i-1\) in turn, the \(V(x_{k+1})\) can be calculated as
$$\begin{aligned}&V(x_{k+1})\\&\quad =\mathrm{E}\Bigg [x_k'\big ((\mathcal {C}_k^0)'(k){\mathcal {P}}_{k+1}^0\mathcal {C}_k^0(k)+(\mathcal {C}_k^0)'(k){\mathcal {P}}_{k+1}^1\\&\qquad +({\mathcal {P}}_{k+1}^1)' \mathcal {C}_k^0(k)\big )x_k+2x_k'\sum \limits _{j=1}^d\big ((\mathcal {C}_k^0)'(k){\mathcal {P}}_{k+1}^0\mathcal {C}_k^j(k)\\&\qquad +(\mathcal {C}_k^0)'(k) {\mathcal {P}}_{k+1}^{j+1}+({\mathcal {P}}_{k+1}^1)'\mathcal {C}_k^j(k)\big )x_{k-j}\\&\qquad +\sum \limits _{j=1}^d\sum \limits _{i=1}^dx_{k-i}' \big ((\mathcal {C}_k^i)'(k) {\mathcal {P}}_{k+1}^0\mathcal {C}_k^j(k)+(\mathcal {C}_k^i)'(k){\mathcal {P}}_{k+1}^{j+1} \\&\qquad +({\mathcal {P}}_{k+1}^{j+1})'\mathcal {C}_k^i(k)\big )x_{k-j}+2u_k'\sum \limits _{j=0}^dN_k^jx_{k-j}+u_k'(\varOmega _k - R)u_k\\&\qquad +\sum \limits _{j=0}^{d-1}\sum \limits _{i=0}^{d-1}\sum \limits _{l=1}^dx_{k-j}' [\mathcal {C}_{j+l}'(k + l){\mathcal {P}}_{k+l+1}^0\mathcal {C}_{i+l}(k + l)\\&\qquad + \gamma \bar{\mathcal {C}}_{j+l}'(k + l){\mathcal {P}}_{k+l+1}^0 \bar{\mathcal {C}}_{i+l}(k + l)+\mathcal {C}_{j+l}'(k + l){\mathcal {P}}_{k+l+1}^{i+l+1}\\&\qquad +({\mathcal {P}}_{k+l+1}^{j+l+1})'\mathcal {C}_{i+l}(k + l)-(N_{k+l}^{j+l})'\varOmega _{k+l}^{-1}N_{k+l}^{i+l}]x_{k-i}\\&\qquad +2\mu _k'\sum \limits _{j=0}^d{\mathcal {P}}_{k+1}^0\mathcal {C}_k^j(k)x_{k-j}+2\mu _k'{\mathcal {P}}_{k+1}^0\mathcal {D}_k(k)u_k+\mu _k'{\mathcal {P}}_{k+1}^0\mu _k \\&\qquad +\sum \limits _{j=0}^d\mu _k'{\mathcal {P}}_{k+1}^{j+1}x_{k-j}+2x_{k+1}'\varPhi _{k+1}\Bigg ]. \end{aligned}$$
Constructing the form \(V(x_k)-V(x_{k+1})\), we have
$$\begin{aligned}&V(x_k)-V(x_{k+1})\\&\quad =\mathrm{E}\Bigg [x_k'Q_kx_k+u_k'R_ku_k - \Bigg (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\Bigg )'\varOmega _k\Bigg (u_k\\&\qquad +\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\Bigg )-2u_k'\varSigma _k+2x_k\varPhi _k-2\sum \limits _{j=0}^dx_{k-j}'\\&\qquad \times \Bigg ((\mathcal {C}_k^j)'(k)\Bigg ({\mathcal {P}}_{k+1}^0\mu _k+\varPhi _{k+1}\Bigg )+{\mathcal {P}}_{k+1}^{j+1}\mu _k\Bigg )-2\mu _k'\varPhi _{k+1}\\&\qquad -\mu _k'{\mathcal {P}}_{k+1}^0\mu _k\Bigg ]. \end{aligned}$$
Denote
$$\begin{aligned} \phi _k^i&=\big (\mathcal {C}_i'(k)-(N_k^i)'\varOmega _k^{-1}\mathcal {D}'(k)\big )({\mathcal {P}}_{k+1}^0{\bar{\mu }}_k+\varPhi _{k+1})\\&\quad +\big (\bar{\mathcal {C}}_i'(k)-(N_k^i)'\varOmega _k^{-1}\bar{\mathcal {D}}'(k)\big ){\mathcal {P}}_{k+1}^0\tau +({\mathcal {P}}_{k+1}^{i+1})'{\bar{\mu }}_k. \end{aligned}$$
We can obviously know that \(\varPhi _k=\sum \limits _{i=0}^d\phi _{k+i}^i\). Then,
$$\begin{aligned}&\mathrm{E}\Bigg [2x_k\varPhi _k - 2\sum \limits _{j=0}^dx_{k-j}'\big ((\mathcal {C}_k^j)'(k)({\mathcal {P}}_{k+1}^0\mu _k + \varPhi _{k+1})\nonumber \\& + {\mathcal {P}}_{k+1}^{j+1}\mu _k\Bigg ]\nonumber \\&\quad =2\sum \limits _{j=0}^dx_{k-j}'\phi _k^j-2\sum \limits _{j=0}^d\big (\phi _k^j+(N_k^j)'\varOmega _k^{-1}\varSigma _k\big )\nonumber \\&\quad =-2\sum \limits _{j=0}^dx_{k-j}'(N_k^j)'\varOmega _k^{-1}\varSigma _k. \end{aligned}$$
(26)
By virtue of (26), the following equation becomes
$$\begin{aligned}&V(x_k)-V(x_{k+1})\\&\quad =\mathrm{E}\Bigg [x_k'Q_kx_k+u_k'R_ku_k - \big (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\big )'\varOmega _k\big (u_k\\&\qquad +\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\big )-2u_k'\varSigma _k-2\sum \limits _{j=0}^dx_{k-j}'(N_k^j)'\varOmega _k^{-1}\varSigma _k\\&\qquad -2\mu _k'\varPhi _{k+1}-\mu _k'{\mathcal {P}}_{k+1}^0\mu _k\Bigg ]\\&\quad =\mathrm{E}\big [x_k'Q_kx_k+u_k'R_ku_k - \big (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\big )'\varOmega _k\big (u_k\\&\qquad +\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\big )-2u_k'\varSigma _k-2\sum \limits _{j=0}^dx_{k-j}'(N_k^j)'\varOmega _k^{-1}\varSigma _k\\&\qquad -\varSigma _k'\varOmega _k^{-1}\varSigma _k+\varSigma _k'\varOmega _k^{-1}\varSigma _k-2\mu _k'\varPhi _{k+1}-\mu _k'{\mathcal {P}}_{k+1}^0\mu _k\big ]\\&\quad =x_k'Q_kx_k + u_k'R_ku_k - \big (u_k + \varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}+\varOmega _k^{-1}\varSigma _k\big )'\\&\qquad \times \varOmega _k\big (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}+\varOmega _k^{-1}\varSigma _k\big ) + \varSigma _k'\varOmega _k^{-1}\varSigma _k\\&\qquad -2{\bar{\mu }}_k'\varPhi _{k+1}-{\rm Tr}[{\mathcal {P}}_{k+1}^0Q_{\mu _k}]. \end{aligned}$$
Adding from \(k=0\) to \(k=N\), the following equation is obtained:
$$\begin{aligned}&V(x_0)-V(x_{N+1})\\&\quad =\sum \limits _{k=0}^{N}\Bigg [x_k'Q_kx_k + u_k'R_ku_k - \Bigg (u_k + \varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}\\&\qquad + \varOmega _k^{-1} \varSigma _k\Bigg )'\varOmega _k\Bigg (u_k + \varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j} + \varOmega _k^{-1}\varSigma _k\Bigg )\\&\qquad + \varSigma _k'\varOmega _k^{-1} \varSigma _k-2{\bar{\mu }}_k'\varPhi _{k+1}-{\rm Tr}[{\mathcal {P}}_{k+1}^0Q_{\mu _k}]\Bigg ]. \end{aligned}$$
Then, the cost function (2) becomes
$$\begin{aligned} J_N&=V(x_0)+\sum \limits _{k=0}^N\Bigg [\Bigg (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}+\varOmega _k^{-1}\varSigma _k\Bigg )'\\&\quad \times \varOmega _k\Bigg (u_k+\varOmega _k^{-1}\sum \limits _{j=0}^dN_k^jx_{k-j}+\varOmega _k^{-1}\varSigma _k\Bigg )\\&\quad -\varSigma _k'\varOmega _k^{-1} \varSigma _k+2{\bar{\mu }}_k'\varPhi _{k+1}+{\rm Tr}[{\mathcal {P}}_{k+1}^0Q_{\mu _k}]\Bigg ]. \end{aligned}$$
As \(\varOmega _k>0\), the unique optimal controller is
$$\begin{aligned} u_k^*=-\varOmega _k^{-1}\sum \limits _{j=0}^dN_{k}^jx_{k-j}-\varOmega _k^{-1}\varSigma _k, \end{aligned}$$
which minimized the cost function (2), and the optimal cost is
$$\begin{aligned} J_N^*=V(x_0) + \sum \limits _{k=0}^{N}\Bigg (2{\bar{\mu }}_k'\varPhi _{k+1} - \varSigma _k'\varOmega _k^{-1}\varSigma _k+ {\rm Tr}[{\mathcal {P}}_{k+1}^0Q_{g_k}]\Bigg ). \end{aligned}$$
Now, the proof of Theorem 1 is completed.
