\documentclass[reqno]{amsart}
\usepackage{hyperref}
\AtBeginDocument{{\noindent\small
\emph{Electronic Journal of Differential Equations},
Vol. 2010(2010), No. 07, pp. 1--19.\newline
ISSN: 1072-6691. URL: http://ejde.math.txstate.edu or http://ejde.math.unt.edu
\newline ftp ejde.math.txstate.edu}
\thanks{\copyright 2010 Texas State University - San Marcos.}
\vspace{9mm}}
\begin{document}
\title[\hfilneg EJDE-2010/07\hfil Orbital stability of periodic solutions]
{Orbital stability of periodic travelling waves for coupled
nonlinear Schr\"odinger equations}
\author[Ademir Pastor\hfil EJDE-2010/07\hfilneg]
{Ademir Pastor}
\address{Ademir Pastor Ferreira\newline
IMPA, Estrada Dona Castorina 110 \\
CEP 22460-320, Rio de Janeiro, RJ, Brazil}
\email{apastor@impa.br}
\thanks{Submitted July 26, 2008. Published January 13, 2010.}
\subjclass[2000]{76B25, 35Q55, 35Q51}
\keywords{Schr\"odinger equation; periodic travelling waves;
orbital stability}
\begin{abstract}
This article addresses orbital stability of periodic
travelling-wave solutions for coupled nonlinear Schr\"odinger
equations. We prove the existence of smooth curves of
periodic travelling-wave solutions depending on the dnoidal-type
functions. Orbital stability analysis is developed in the context
of Hamiltonian systems. We consider both the stability problem
by periodic perturbations which have the same fundamental period
as the corresponding periodic wave and the stability problem by
periodic perturbations having two or more times the minimal period
as the corresponding periodic wave.
\end{abstract}
\maketitle
\numberwithin{equation}{section}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{remark}[theorem]{Remark}
\section{Introduction}
This paper is mainly concerned with the orbital stability of
periodic travelling-wave solutions for the following coupled
nonlinear Schr\"odinger equations
\begin{equation} \label{1.1}
\begin{gathered}
iu_t+ru_{xx}+\Big(\eta|u|^2 +\sigma |w|^2 \Big)u = 0 \\
i\rho w_t+sw_{xx}+\Big(\frac{1}{\eta} |w|^2+
\sigma|u|^2 \Big)w = 0,
\end{gathered}
\end{equation}
where $u$ and $w$ are complex-valued functions of the variables
$x,t \in \mathbb{R}$, the parameters $\rho, \sigma$ and $\eta$ are
positive real constants, and $r=\pm 1$, $s=\pm 1$.
In optics, \eqref{1.1} describes the interaction between
two waves of different frequencies $\omega_1$ and $\omega_2$ or two
waves of the same frequency $\omega$ but belonging to two different
polarizations. The parameters $\eta$ and $\rho$ produce an effective
asymmetry between the modes for the case of the interaction between
waves of the different frequencies. Here, $\sigma$ is the parameter
of the cross-phase modulation, which may be determined in terms of
the parameters of the corresponding physical problem (see
\cite{Agrawal,Kivshar2,Menyuk}), and $r,s$ describe
the type of the group-velocity dispersion. For two waves of
different frequencies $\omega_1$ and $\omega_2$, one usually has
$\eta=\omega_1^2/\omega_2^2$ and $\sigma=2$; for two waves of
different polarizations in a birefringent optical medium one has
$\eta=1$ and $\sigma=2/3$ (see e.g. \cite{Agrawal}).
More generally, \eqref{1.1} is a particular case of the
system
\begin{equation} \label{1.2}
\begin{gathered}
iu_t+ru_{xx}-\alpha_1 u+\Big(\eta|u|^2
+\sigma |w|^2 \Big)u+\beta_1\overline{u}^2w = 0 \\
i\rho w_t+sw_{xx}-\alpha_2 w+\Big(\frac{1}{\eta}
|w|^2 +\sigma|u|^2 \Big)w+\beta_2 u^3 = 0,
\end{gathered}
\end{equation}
which arises in many physical situations. For instance, when
$r=s=1$, $\alpha_1=1$, $\eta=1/9$, $\sigma=2$, $\beta_1=1/3$, and
$\beta_2=1/9$, the system \eqref{1.2} reads as
\begin{equation} \label{1.3}
\begin{gathered}
iu_t+u_{xx}-u+\Big(\frac{1}{9}|u|^2 +2 |w|^2 \Big)u
+\frac{1}{3}\overline{u}^2w = 0 \\
i\rho w_t+w_{xx}-\alpha_2 w+\Big(9 |w|^2 +2|u|^2
\Big)w+\frac{1}{9}u^3 = 0,
\end{gathered}
\end{equation}
which describes the resonant interaction between a linearly
polarized beam of frequency $\omega$ and its third harmonic (see
\cite{Sammut1,Sammut}).
Here, we specialize the system \eqref{1.1} in the case where
$\rho=1$ and $r=s=1$, but we permit all values of $\eta, \sigma>0$.
Thus, \eqref{1.1} reduces to
\begin{equation} \label{1.4}
\begin{gathered}
iu_t+u_{xx}+\Big(\eta|u|^2 +\sigma |w|^2 \Big)u = 0 \\
i w_t+w_{xx}+\Big(\frac{1}{\eta} |w|^2+ \sigma|u|^2
\Big)w = 0.
\end{gathered}
\end{equation}
From the mathematical viewpoint, \eqref{1.4} has been
studied by many authors (see e.g.
\cite{Ambrosetti,deFigueiredo,Lin,Lin1,Sirakov},
\cite{Ohta}--\cite{Pelinovsky1}, but only in the context of
existence and stability of solitary-wave solutions. As far as we
know, no results concerning the stability of periodic
travelling-wave solutions have been shown.
As a matter of fact, only a few papers address orbital stability of
periodic travelling-wave solutions for Schr\"odinger-type systems.
We cite a few known. In \cite{Angulo2}, the authors considered a
system arising in nonlinear optics (in a medium with quadratic
nonlinearities, see \cite{Kivshar}) and they showed the existence
and stability/instability of periodic travelling waves depending on
the Jacobian elliptic function of the cnoidal type. In
\cite{Angulo5,Pastor}, the authors considered the system
\eqref{1.3}. The existence and stability/instability of periodic
travelling waves depending on the \emph{dnoidal} (in \cite{Angulo5})
and \emph{cnoidal} (in \cite{Pastor}) functions were shown. The
techniques to obtain such results were the ones developed by
Grillakis, Shatah, and Strauss \cite {Grillakis4}, and Grillakis
\cite{Grillakis2}.
For the single cubic Schr\"odinger equation
\begin{equation} \label{Schrodinger}
iu_t+u_{xx}+|u|^2u=0,
\end{equation}
Angulo \cite{Angulo1} established the existence of periodic
travelling waves based on the dnoidal-type functions. By combining
the classical Lyapunov method and the Floquet theory associated to
the Lam\'e equation
$$
v'' + [\lambda-6k^2sn^2(x;k)]v=0,
$$
the author showed their orbital stability by periodic perturbations
which have the same fundamental period as the corresponding dnoidal
wave (note that one can also apply the theory in
\cite{Grillakis4}), and orbital instability by periodic
perturbations with twice the fundamental period of the dnoidal wave.
As evidenced above, the abstract \emph{Stability/Instability Theorem}
in \cite{Grillakis4} can be applied for many dispersive equations.
However, the main difficulty when one works with travelling waves for
coupled systems, instead of one single equation, is that the
spectral analysis for the ``linearized Hamiltonian'' turns out to be
a more delicate matter. Indeed, in this case one needs to deal with
a matrix having Schr\"odinger-type operators as components. As a
consequence, in many examples, the Stability/Instability criterium
in \cite{Grillakis4} turns out to be insufficient for a complete
stability/instability analysis of the travelling waves. The main
reason for this, is that in such approach one needs to know the
exact number of negative eigenvalues of the linearized Hamiltonian.
Grillakis \cite{Grillakis1,Grillakis2} obtained others special
instability theorems, which get orbital instability from the linear
instability of the zero solution for the linearization of the system
around the orbit generated by the corresponding travelling wave. The
main advantage when one uses the Grillakis approach is that one does
not need to know the exact number of negative eigenvalues of the
linearized Hamiltonian, but only to have an estimate on a certain
bound (see Subsection \ref{subsection4.3} for the details).
Now, we turn our attention to the structure of the paper. The
periodic travelling-wave solutions we are interested in are of the
form
\begin{equation} \label{1.5}
u(x,t)=e^{i\gamma t}\phi_{\gamma}(x), \quad w(x,t)=e^{i\gamma
t}\psi_{\gamma}(x),
\end{equation}
where $\phi_\gamma, \psi_\gamma: \mathbb{R}\to \mathbb{R}$
are smooth periodic functions with the same fixed period $L>0$ and
$\gamma$ is a real parameter. Substituting \eqref{1.5} into
\eqref{1.4}, we get the following system of ordinary differential
equations
\begin{equation} \label{1.6}
\begin{gathered}
\phi_\gamma''-\gamma\phi_\gamma+\Big(\eta\phi_\gamma^2+\sigma\psi_\gamma^2 \Big)\phi_\gamma= 0 \\
\psi_\gamma''-\gamma\psi_\gamma +\Big(\frac{1}{\eta} \psi_\gamma^2
+\sigma\phi_\gamma^2 \Big)\psi_\gamma = 0.
\end{gathered}
\end{equation}
It is well known that \eqref{1.6} admits solitary-wave
solutions (for $\eta=1$) of the form
\begin{equation}\label{solitary}
\phi_\gamma(x)=\psi_\gamma(x)=
\sqrt{\frac{2\gamma}{\sigma+1}}\mathop{\rm sech}(\sqrt\gamma x),
\quad \gamma>0.
\end{equation}
In \cite{Pelinovsky}, the authors proved that the waves in
\eqref{solitary} are linearly stable for $\sigma>0$ and linearly
unstable for $-1-1$.
In the present paper, we consider two classes of periodic solutions.
First, we suppose $\psi_\gamma \equiv 0$. Then, we can find a smooth
curve of periodic solutions for \eqref{1.6} depending on the dnoidal
type function, namely,
\begin{equation} \label{1.7}
\gamma \in \Big(\frac{2\pi^2}{L^2}, +\infty \Big) \mapsto
(\phi_{\gamma}, 0) \in H_{\rm per}^m([0,L])\times H_{\rm per}^n([0,L]),
\end{equation}
where
\begin{equation} \label{1.8}
\phi_\gamma(x)=\eta_1dn
\Big(\frac{\sqrt\eta}{\sqrt2}\eta_1x;k\Big), \quad
k^2=\frac{\eta_1^2-\eta_2^2}{\eta_1^2},
\end{equation}
and $\eta_1, \eta_2$ are smooth functions depending on the parameter
$\gamma$ with $0\sigma$, and the orbit $\mathcal{O}_{2L}=
\{(e^{ir}\phi_\gamma(\cdot),0);\,\, r \in \mathbb{R} \}$ is
unstable in the space $H^1_{\rm per}([0,2L])\times H^1_{\rm per}([0,2L])$.
On the other hand, for the non-semitrivial solutions \eqref{1.9}
(with $\eta=b=1)$, we show that the orbit $\widetilde{\mathcal{O}}=
\{(e^{ir}\phi_\gamma(\cdot),e^{ir}\phi_\gamma(\cdot));\,\, r \in
\mathbb{R} \}$ is spectrally stable with respect to periodic
perturbations having the same fundamental period of $\phi_\gamma$,
and it is orbitally unstable in the space $H^1_{\rm per}([0,2L])\times
H^1_{\rm per}([0,2L])$. Moreover, if we assume $-11/2$ then $ X_{s,b}
\hookrightarrow C(\mathbb{R}_t; H^s_{\rm per}) $.
\end{remark}
Let $\zeta \in C_0^\infty(\mathbb{R})$ be a cut off function such
that $\mathop{\rm supp} \zeta \subset (-2,2)$ and $\zeta \equiv 1$ on the
interval $[-1,1]$. For each $T>0$, we define
$\zeta_T(t)=\zeta(t/T)$.
\begin{lemma} \label{lemma2.1}
Let $s\in \mathbb{R}$, $b \in (1/2,1)$ and $T \in (0,1]$. Then,
\begin{enumerate}
\item[(i)] ${ \| \zeta_T U(t)v
\|_{X_{s,b}} \leq c \|v \|_{H_{\rm per}^s}}$,
\item[(ii)] $ \big\| \zeta_T \int_0^t U(t-t')f(t')dt' \big\|_{X_{s,b}}
\leq c T ^\gamma \|f\|_{X_{s,b-1}} $,
where $\gamma$ is a positive constant.
\end{enumerate}
\end{lemma}
For a proof of the above lemma, see for example
Kenig, Ponce, and Vega \cite{Kenig2, Kenig3}.
Next, we have a trilinear estimate, which may be proved following
similar arguments as the ones in Bourgain \cite{Bourgain};
see also \cite{Angulo5,Bourgain1}.
\begin{lemma} \label{lemma2.2}
Let $s \geq 0$ and $b \in (3/8, 5/8)$. Then,
\[
\|u^{\alpha_1}\overline{u}^{\alpha_2} w^{\alpha_3}
\overline{w}^{\alpha_4} \|_{X_{s,b-1}} \leq c
\|u\|_{X_{s,b}}^{\alpha_1+\alpha_2}
\|w\|_{X_{s,b}}^{\alpha_3+\alpha_4},
\]
where $\alpha_1,\alpha_2,\alpha_3,\alpha_4 \in \{0,1,2,3 \}$ with
$\alpha_1+\alpha_2+\alpha_3+\alpha_4=3$.
\end{lemma}
With the above lemmas in hand, we are able to prove our local
well-posedness result.
\begin{theorem}[Local well-posedness] \label{theorem2.3}
Let $s\geq 0$ and $b \in (1/2,5/8)$. For any $(u_0,w_0) \in H^s_{\rm
per}([0,L]) \times H^s_{\rm per}([0,L])$, there exist
$T=T(\|(u_0,w_0) \|_{H^s_{\rm per} \times H^s_{\rm per}})>0$ and a
unique solution $(u(t),w(t))$ of the initial-value problem
\eqref{2.1} satisfying
\begin{gather*}
(u,w) \in C([-T,T]; H^s_{\rm per}([0,L]) \times H^s_{\rm per}([0,L])), \\
(\zeta_T u, \zeta_T w) \in X_{{s,b}} \times X_{{s,b}}.
\end{gather*}
Moreover, given $T'\in (0,T)$, there exists a neighborhood
$\mathcal{W}$ of $(u_0,w_0)$ in $H^s_{\rm per} \times H^s_{\rm per}$ such
that the map $(u_0,w_0) \mapsto (u(t),w(t))$ from $\mathcal{W}$
into $C([-T',T']; H^s_{\rm per}\times H^s_{\rm per})$ is Lipschitz
continuous.
\end{theorem}
\begin{proof}[Sketch of the proof]
We define the metric space of functions
$$
\mathcal{X}_M =\{ (u,w) \in X_{{s,b}} \times X_{{s,b}};
\|(u,w)\|_{X_{{s,b}} \times X_{{s,b}}}:=
\|u\|_{X_{s,b}}+\|w\|_{X_{s,b}} \leq M \},
$$
and the map $\Phi=(\Phi_1,\Phi_2)$, where
\begin{equation} \label{2.2}
\begin{gathered}
\Phi_1(u,w)(t)= \zeta_T(t)U(t)u_0+i\zeta_T(t)
\int_0^t U(t-t')\Big(\eta|u|^2u +\sigma |w|^2u \Big)(t') dt',
\\
\Phi_2(u,w)(t)= \zeta_T(t)U(t)w_0+i\zeta_T(t)
\int_0^t U(t-t')\Big(\frac{1}{\eta} |w|^2w+ \sigma|u|^2w \Big)(t')
dt',
\end{gathered}
\end{equation}
where $M>0$ and $T \in (0,1]$. By choosing $M$ and $T$ suitably, and
using Lemmas \ref{lemma2.1} and \ref{lemma2.2}, we can prove that
$\Phi:\mathcal{X}_M\to \mathcal{X}_M$ is a contraction.
Hence, the contraction principle implies the existence of a unique
fixed point for the integral equations \eqref{2.2}, which solves our
problem. The rest of the proof follows standard arguments, which
will be omitted.
\end{proof}
Finally, we can establish our global well-posedness result.
\begin{theorem}[Global well-posedness] \label{theorem2.4}
For $s \geq 0$ and $(u_0,w_0) \in H^s_{\rm per}([0,L]) \times
H^s_{\rm per}([0,L])$, the solution $(u(t),w(t))$ given in Theorem
\ref{theorem2.3} can be extend to any interval of time.
\end{theorem}
\begin{proof}
This follows from the conserved quantity
\[
\int (|u(x,t)|^2+|w(x,t)|^2)dx = \int (|u_0(x)|^2+ |w_0(x)|^2)dx,
\]
and \emph{a priori} estimates (see e.g. \cite{Bourgain1}).
\end{proof}
\begin{remark} \label{rmk2.7} \rm
The same ideas used to prove Theorems \ref{theorem2.3} and
\ref{theorem2.4} can be applied to show local and global
well-posedness results for the system \eqref{1.2} (see
\cite{Angulo5}).
\end{remark}
\section{Existence of smooth curves of dnoidal waves}
\label{existence}
This section is devoted to establishing the existence of smooth
curves of periodic travelling-wave solutions for the system
\eqref{1.4} having the form
\begin{equation} \label{3.1}
u(x,t)=e^{i\gamma t}\phi_{\gamma}(x), \quad
w(x,t)=e^{i\gamma t}\psi_{\gamma}(x),
\end{equation}
where $\phi_\gamma, \psi_\gamma: \mathbb{R} \to \mathbb{R}$
are smooth periodic functions with the same fixed period $L>0$ and
$\gamma$ is a real parameter. Thus, $\psi_\gamma=\psi$ and
$\phi_\gamma=\phi$ must satisfy the system of ordinary differential
equations
\begin{equation} \label{3.2}
\begin{gathered}
\phi''-\gamma\phi+\Big(\eta\phi^2+\sigma\psi^2 \Big)\phi = 0 \\
\psi''- \gamma\psi +\Big(\frac{1}{\eta} \psi^2 +\sigma\phi^2
\Big)\psi = 0.
\end{gathered}
\end{equation}
To solve this system, we consider $\psi=b\phi$, for
some real constant $b$, and analyze two cases.
\noindent\textbf{Case 1 (semitrivial solutions)}: $b=0$.
In this case, \eqref{3.2} reduces to the
differential equation
\begin{equation} \label{3.3}
\phi''- \gamma\phi+\eta\phi^3=0.
\end{equation}
It is well known that this equation has a positive solution of the
form
\begin{equation} \label{3.4}
\phi(x)=\eta_1 dn \Big(\frac{\sqrt\eta}{\sqrt2}\eta_1x;k \Big),
\quad k^2=\frac{\eta_1^2-\eta_2^2}{\eta_1^2},
\end{equation}
where $\eta_1, \eta_2$ are real constants satisfying
$\eta_1>\eta_2>0$.
In the sequel, we prove that the parameters $\eta_1,\eta_2$ can be
chosen such that the function $\phi$ in \eqref{3.4} has fundamental
period $L$ (here we give only the main ingredients, for details see
\cite{Angulo1},\cite{Angulo5}). Indeed, it is easy to see that
parameters $\eta_1$, $\eta_2$ must satisfy
\begin{equation} \label{3.5}
\begin{gathered}
\eta_1^2+\eta_2^2=\frac{2\gamma}{\eta}=2\omega \\
-\eta_1^2\eta_2^2= \frac{4}{\eta}B_\phi,
\end{gathered}
\end{equation}
where $\omega:=\gamma/\eta$ and $B_\phi$ is an integration constant.
Therefore, for a fixed $\omega>0$, we get from \eqref{3.5} that
$00$ is fixed), namely,
$$
T_{\phi}(\eta_2)=\frac{2\sqrt{2}}{\sqrt\eta
\sqrt{2\omega-\eta_2^2}}K(k(\eta_2)), \quad
k^2(\eta_2)=\frac{2\omega-2\eta_2^2}{2\omega-\eta_2^2}.
$$
But, since $T_{\psi}(\eta_2) \to \infty$ as $\eta_2
\to 0^+$, $T_{\psi}(\eta_2) \to \frac{\pi
\sqrt{2}}{\sqrt{\omega \eta}}$ as $\eta_2 \to
\sqrt{\omega}$, and the function $\eta_2 \in (0, \sqrt{\omega})
\mapsto T_{\phi}(\eta_2)$ is strictly decreasing, we conclude that
$T_\phi> \frac{\pi \sqrt{2}}{\sqrt{\omega \eta}}$.
Hence, given $L>0$, by fixing $\omega>0$ such that
$\sqrt\omega>\frac{\pi\sqrt2}{L\sqrt\eta}$, there is a unique
$\eta_2=\eta_2(\omega) \in (0,\sqrt\omega)$ such that the dnoidal
wave $\phi$ in \eqref{3.4} has fundamental period $L=T_\phi$.
In addition, we can construct, for each $L>0$ fixed, a smooth
curve of dnoidal-wave solutions depending on the parameter $\omega$
(and hence on $\gamma$) such that each element of the curve has
fundamental period $L$. More precisely, we prove the following.
\begin{theorem} \label{theorem3.1}
Let $L>0$ be fixed. The following statements hold.
\begin{itemize}
\item[(i)] There is a smooth function
$\Gamma:\big(\frac{2\pi^2}{\eta L^2}, +\infty \big)
\to \mathbb{R}$ such that
$$
\frac{2\sqrt{2}}{\sqrt\eta \sqrt{2\omega-\eta_2^2}}K(k)=L,
$$
where $\eta_2=\Gamma(\omega)$ and
$k^2=k^2(\omega)=\frac{2\omega-2\eta_2^2}{2\omega-\eta_2^2}$.
\item[(ii)] The function $\Gamma$ in \textrm{(i)} is strictly decreasing
and the modulus $k=k(\omega)$ satisfies $\frac{dk}{d\omega}>0$.
\item[(iii)] For every $\gamma \in
\big(\frac{2\pi^2}{L^2}, +\infty \big)$ and
$\omega=\omega(\gamma)=\gamma/\eta$, the dnoidal wave defined in
\eqref{3.4},
$$
\phi_\gamma(x):=\phi_{\omega(\gamma)}(x)=\sqrt{2\omega-\eta_2^2}
\, dn \Big(\frac{\sqrt{\eta(2\omega-\eta_2^2)}}{\sqrt2}x;k
\Big)
$$
has fundamental period $L$ and satisfies \eqref{3.3}. Moreover, the
mapping
$$
\gamma \in \big(\frac{2\pi^2}{L^2}, +\infty \big) \mapsto
\phi_{\gamma} \in H_{\rm per}^n([0,L])
$$
is a smooth function.
\end{itemize}
\end{theorem}
\begin{proof}
The proof is an application of the Implicit
Function Theorem. We refer to \cite{Angulo1} or \cite{Angulo5} for
the details.
\end{proof}
\noindent \textbf{Case 2 (non-semitrivial solutions)}: $b\neq0$.
In this case, from \eqref{3.2}, we get the system
\begin{equation} \label{3.6}
\begin{gathered}
\phi''-\gamma\phi+\big(\eta+\sigma b^2 \big)\phi^3 = 0 \\
\phi''- \gamma\phi +\big(\frac{b^2}{\eta}+ \sigma \big)\phi^3 = 0.
\end{gathered}
\end{equation}
To solve this system, we make the following assumptions:
\begin{itemize}
\item[(H1)] $\eta+\sigma b^2=\frac{b^2}{\eta}+\sigma$,
\item[(H2)]$(\sigma-\eta)(\sigma-\frac{1}{\eta})>0$.
\end{itemize}
These two assumptions allows us to reduce \eqref{3.6} to
the single equation
\begin{equation} \label{3.7}
\phi''-\gamma \phi+ \theta\phi^3 = 0,
\end{equation}
where $\theta=\eta+\sigma b^2>0$ is a real constant. As in Case 1,
the equation \eqref{3.7} admits a dnoidal-wave solution
\begin{equation} \label{3.8}
\phi(x)=\theta_1 dn \Big(\frac{\theta_1\sqrt{\theta}}{\sqrt{2}}x; k
\Big), \quad
k^2=\frac{\theta_1^2-\theta_2^2}{\theta_1^2},
\end{equation}
where $\theta_1, \theta_2$ are real constants and satisfy
$\theta_1^2+\theta_2^2=2\omega$, where in this case we have written
$\omega=\gamma/\theta$.
By similar arguments as in Case 1, we may verify that for every
$L>0$ and $\omega>0$ such that $\sqrt{\omega}>\frac{\pi
\sqrt{2}}{\sqrt{\theta}L}$, there exists a unique
$\theta_2=\theta_2(\omega) \in (0,\sqrt{\omega})$ such that the
dnoidal wave $\phi=\phi(\cdot;\theta_1(\omega); \theta_2(\omega))$,
given in \eqref{3.8}, has fundamental period $L=T_{\phi}$.
\begin{remark} \label{remark3.4} \rm
Formally, the periodic-wave solution \eqref{3.8} contains the
solitary-wave solution \eqref{solitary}. Indeed, as $\theta_2 \to
0^+$ it follows that $\theta_1 \to \sqrt{2\gamma/(\eta+\sigma b^2)}$
and $dn(\cdot;1^-)\sim \mathop{\rm sech}(\cdot)$. Thus,
$$
\phi(x) \sim \sqrt{\frac{2\gamma}{\eta+\sigma b^2}}
\mathop{\rm sech}(\sqrt\gamma x),
$$
\end{remark}
As in Theorem \ref{theorem3.1}, we can prove the following.
\begin{theorem} \label{theorem3.2}
Let $L>0$ be fixed. The following statements hold.
\begin{itemize}
\item[(i)] There exists a smooth function
$\Lambda:\big(\frac{2\pi^2}{\theta L^2}, +\infty \big)
\to \mathbb{R}$ such that
$$
\frac{2\sqrt{2}}{\sqrt\theta\sqrt{2\omega-\theta_2^2}}K(k)=L,
$$
where $\theta_2=\Lambda(\omega)$ and
$k^2=k^2(\omega)=\frac{2\omega-2\theta_2^2}{2\omega-\theta_2^2}$.
\item[(ii)] The function $\Lambda$ in \textrm{(i)} is strictly decreasing
and the modulus $k=k(\omega)$ satisfies $\frac{dk}{d\omega}>0$.
\item[(iii)] For every $\gamma \in
\big(\frac{2\pi^2}{L^2}, +\infty \big)$ and
$\omega=\omega(\gamma)=\gamma/\theta$, the dnoidal wave defined in
\eqref{3.8},
$$
\phi_\gamma(x)=\phi_{\omega(\gamma)}(x)=
\sqrt{2\omega-\theta_2^2}\, dn
\Big(\frac{\sqrt{\theta(2\omega-\theta_2^2)}}{\sqrt2}x; k \Big)
$$
has fundamental period $L$ and satisfies \eqref{3.7}. Moreover, the
mapping
$$
\gamma \in \Big(\frac{2\pi^2}{L^2}, +\infty \Big) \mapsto
\phi_{\gamma} \in H_{\rm per}^n([0,L])
$$
is a smooth function.
\end{itemize}
\end{theorem}
\section{Orbital Stability/Instability of semitrivial solutions}
\label{section4}
In this section, we prove our results concerning the orbital
stability/instability of the semitri\-vial solutions given in
Theorem \ref{theorem3.1}. First, we note that the system \eqref{1.4}
may be written as a Hamiltonian system. Indeed, by writing $u=P+iQ$,
$w=R+iS$, and $U=(P,R,Q,S)$, we rewrite \eqref{1.4} as
\begin{equation} \label{Hamiltonian}
\frac{\partial U}{\partial t}(t)=J\mathcal{H}'(U(t)),
\end{equation}
where $J$ is the skew-symmetric matrix
\begin{equation} \label{4.1}
J= \begin{pmatrix} 0 & 0 & 1 & 0 \\
0 & 0 & 0 & 1 \\
-1 & 0 & 0 & 0 \\
0 & -1 & 0 & 0
\end{pmatrix},
\end{equation}
and $\mathcal{H}$ is the energy functional
\begin{equation} \label{4.2}
\begin{aligned}
\mathcal{H}(P,R,Q,S)
&= \frac{1}{2}\int
\Big\{(P_x^2+Q_x^2)+(R_x^2+S_x^2)-\frac{\eta}{2}(P^2+Q^2)^2{}
-\frac{1}{2\eta}(R^2+S^2)^2 {} \\
&\quad -\sigma(P^2+Q^2)(R^2+S^2) \Big\} dx.
\end{aligned}
\end{equation}
In \eqref{Hamiltonian}, $\mathcal{H}'$ denotes the Fr\'echet
derivative of $\mathcal{H}$.
Recall that by using this notation, the functional $\mathcal{F}$ in
\eqref{F} reads as
\[
\mathcal{F}(P,R,Q,S) = \frac{1}{2}\int \big\{(P^2+Q^2)+(R^2+S^2)
\big\} dx.
\]
\subsection{Spectral Analysis}
We consider $L>0$ fixed and define $\Phi=(\phi_\gamma,0,0,0)$, where
$\phi=\phi_\gamma$ is the dnoidal wave given by Theorem
\ref{theorem3.1}. Next, we consider the linearized operator
\begin{equation} \label{4.3}
\mathcal{L_{\gamma}}=\mathcal{H}''(\Phi)+\gamma \mathcal{F}''(\Phi)=
\begin{pmatrix}
\mathcal{L}_{R} & 0 \\
0 & \mathcal{L}_{I}
\end{pmatrix},
\end{equation}
where $\mathcal{L}_{R}$ and $\mathcal{L}_{I}$ are $2\times 2$ matrix
operators defined by
\begin{equation} \label{4.4}
\mathcal{L}_{R} = \begin{pmatrix}
\mathcal{L}_1 & 0 \\
0 & \mathcal{L}_3
\end{pmatrix}, \quad
\mathcal{L}_{I} = \begin{pmatrix}
\mathcal{L}_2 & 0 \\
0 & \mathcal{L}_3
\end{pmatrix},
\end{equation}
with
\begin{equation} \label{4.5}
\mathcal{L}_{1} = -\frac{d^2}{dx^2}+ \gamma-3\eta\phi^2, \quad
\mathcal{L}_{2} = -\frac{d^2}{dx^2}+ \gamma-\eta\phi^2, \quad
\mathcal{L}_{3} = -\frac{d^2}{dx^2}+\gamma -\sigma\phi^2.
\end{equation}
In the sequel, we study the spectrum of the diagonal operator
$\mathcal{L}_\gamma$.
\begin{theorem} \label{theorem4.1}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Consider the operator $\mathcal{L}_\gamma$ in
\eqref{4.3} defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$. The following statements hold.
\begin{itemize}
\item [(i)] If $n(\mathcal{L}_\gamma)$ denotes the number of
negative eigenvalues of $\mathcal{L}_\gamma$ (counting multiplicities), then
$n(\mathcal{L}_\gamma)=2k+1$, for some $k \in \mathbb{N}\cup\{0\}$. Moreover, the remainder
of the spectrum is constituted by a discrete set of
eigenvalues.
\item [(ii)] The kernel of $\mathcal{L}_\gamma$,
$\ker(\mathcal{L}_\gamma)$, is at least two-dimensional and
contains the space spanned by $(\phi',0,0,0)$ and
$(0,0,\phi,0)$.
\end{itemize}
\end{theorem}
To prove Theorem \ref{theorem4.1}, the following lemma is
fundamental.
\begin{lemma} \label{lemma4.2}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Then the following spectral properties hold.
\begin{itemize}
\item [(i)] The operator $\mathcal{L}_{1}$ in $(\ref{4.5})$ defined
in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$ has exactly one negative
eigenvalue which is simple; zero is an eigenvalue which is
simple with eigenfunction $\phi'$. Moreover, the remainder
of the spectrum is constituted by a discrete set of
eigenvalues.
\item [(ii)] The operator $\mathcal{L}_{2}$ in $(\ref{4.5})$
defined in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$
has no negative eigenvalues; zero is a simple eigenvalue
with eigenfunction $\phi$. Moreover, the remainder
of the spectrum is constituted by a discrete set of
eigenvalues.
\end{itemize}
\end{lemma}
\begin{proof}
The proof is essentially the same one as in
\cite[Theorems 3.1 and 3.2]{Angulo1} and
\cite[Theorem 4.1]{Angulo5}, with obvious
modifications. Note that part (ii) is an immediate consequence of
the Floquet theory (see e.g. \cite{Magnus}), since
$\mathcal{L}_{2}\phi=0$ and $\phi$ has no zeros in $[0,L]$.
\end{proof}
\begin{proof}[Proof of Theorem \ref{theorem4.1}]
Part (ii) follows immediately from Lemma \ref{lemma4.2}. To prove
part (i), we first note that, from Lemma \ref{lemma4.2},
$\mathcal{L}_\gamma$ always has a negative eigenvalue (which comes
from the operator $\mathcal{L}_1$). Moreover, from the definition of
$\mathcal{L}_\gamma$, we see that if $\lambda$ is an eigenvalue of
$\mathcal{L}_3$ (defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$) then it is a double eigenvalue of
$\mathcal{L}_\gamma$. Therefore, since $\mathcal{L}_2$ has no
negative eigenvalues, we have proved the theorem.
\end{proof}
\begin{corollary} \label{corollary4.3}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.1}. Consider the operator $\mathcal{L}_\gamma$ in
\eqref{4.3} defined in $L^2_{\rm per}([0,L])$ with domain
$H_{\rm per}^2([0,L])$ and suppose that $\eta>\sigma$. Then,
\begin{itemize}
\item [(i)] $n(\mathcal{L}_\gamma)=1$.
\item [(ii)] $\ker(\mathcal{L}_\gamma)$ is two-dimensional and
spanned by $(\phi',0,0,0)$ and
$(0,0,\phi,0)$.
\end{itemize}
\end{corollary}
\begin{proof}
Since $\eta>\sigma$, we have $\gamma-\eta\phi^2(x)\eta$ then, from Lemma \ref{lemma4.2} and the
Comparison Theorem, we obtain that $n(\mathcal{L}_\gamma)\geq 3$.
Moreover, if $3\eta>\sigma>\eta$ then $n(\mathcal{L}_\gamma)= 3$,
and $\ker(\mathcal{L}_\gamma)$ is
spanned by $(\phi',0,0,0)$ and
$(0,0,\phi,0)$.
\end{enumerate}
With the goal of proving a stability/instability result by periodic
perturbation having twice the wavelength of $\phi_\gamma$, we now
prove the following.
\begin{theorem} \label{theorem4.4}
Let $\phi_\gamma=\phi$ be the dnoidal wave given by Theorem
$\ref{theorem3.1}$.
\begin{itemize}
\item[(i)] Consider the operator $\mathcal{L}_R$ in $(\ref{4.4})$
defined in $L^2_{\rm per}([0,2L])$ with domain $H_{\rm per}^2([0,2L])$. If
$\widetilde{n}(\mathcal{L}_R)$ denotes the number of negative
eigenvalues of $\mathcal{L}_R$ (counting multiplicities), then
$\widetilde{n}(\mathcal{L}_R)=k_0+3$ for some $k_0\in \mathbb{N}$.
Moreover, there exist $k_0+2$ eigenvalues such that the
corresponding eigenfunctions are orthogonal to
$\ker(\mathcal{L}_I)$.
\item[(ii)] Consider the operator $\mathcal{L}_I$ in $(\ref{4.4})$
defined in $L^2_{\rm per}([0,2L])$ with domain $H_{\rm per}^2([0,2L])$. If
$\widetilde{n}(\mathcal{L}_I)$ denotes the number of negative
eigenvalues of $\mathcal{L}_I$ (counting multiplicities), then
$\widetilde{n}(\mathcal{L}_I)=k_0$. Moreover, all the eigenfunctions
corresponding to negative eigenvalues are orthogonal to
$\ker(\mathcal{L}_R)$.
\end{itemize}
\end{theorem}
\begin{proof}
(i) Let $k_0$ be the number of negative eigenvalues of the operator
$\mathcal{L}_3$ (defined in $L^2_{\rm per}([0,2L])$ with domain
$H_{\rm per}^2([0,2L])$). So, for the first part, it suffices to show
that the operator $\mathcal{L}_1$ (defined in $L^2_{\rm per}([0,2L])$
with domain $H_{\rm per}^2([0,2L])$) has exactly 3 negative eigenvalues.
To do this, we consider the semi-periodic eigenvalue problem
\begin{equation} \label{4.6}
\begin{gathered}
\mathcal{L}_{1}\chi=\mu \chi \\
\chi(0)=-\chi(L), \quad \chi'(0)=-\chi'(L).
\end{gathered}
\end{equation}
which is equivalent (under the transformation
$\Lambda(x)=\chi(\alpha x)$,
$\alpha=\frac{\sqrt2}{\eta_1\sqrt\eta}$) to the following
semi-periodic eigenvalue problem associated to the Lam\'e equation:
\begin{equation} \label{4.7}
\begin{gathered}
\Lambda''+[\widetilde{\mu} - 6k^2sn^2(x;k)]\Lambda=0 \\
\Lambda(0)=-\Lambda(2K),\quad \Lambda'(0)=-\Lambda'(2K),
\end{gathered}
\end{equation}
where
\begin{equation} \label{4.8}
\widetilde{\mu} =\frac{2}{\eta\eta_1^2}[\mu- \gamma+3\eta\eta_1^2].
\end{equation}
Now, a straightforward calculation shows us that
$\widetilde{\mu}_0=1+k^2$ and $\widetilde{\mu}_1=1+4k^2$ are the
first two eigenvalues to \eqref{4.7} (see \cite{Ince}), which are
simple with eigenfunctions given, respectively, by
$$
\Lambda_{1,sm}(x)=cn(x;k)dn(x;k), \quad
\Lambda_{2,sm}(x)=sn(x;k)dn(x;k).
$$
Thus, from \eqref{4.8} and the Floquet theory, we obtain the first
part.
For the second part, we note that if $\mu_1, \mu_2$ denote the
corresponding negative eigenvalues for \eqref{4.6}, via \eqref{4.8},
then the unique (up to a constant) eigenfunction for $\mathcal{L}_1$
associated to $\mu_i,i=1,2$ is given by
$\chi_i(x)=\Lambda_{i,sm}(\frac{1}{\alpha}x)$, and so $(\chi_i,0)$
is an eigenfunction for $\mathcal{L}_R$ associated to $\mu_i,i=1,2$.
Next, let $(u,v)\in \ker(\mathcal{L}_I)$, with $u\neq 0$. It follows
from Lemma \ref{lemma4.2} that $u=c\phi$ for some real constant $c$.
Since
$$
\int_0^{4K} dn^2(x;k)cn(x;k)dx=\int_0^{4K} dn^2(x;k)sn(x;k)dx=0,
$$
it is easy to see, from the explicit form of $\phi$ and $\chi_i$,
that
$$
\langle (\chi_i,0), (u,v) \rangle_{L^2_{\rm per}([0,2L])}=c
\int_0^{2L}\chi_i(x) \phi(x) dx=0, \quad i=1,2.
$$
Furthermore, since $\mathcal{L}_3$ is a self-adjoint operator all
its eigenfunctions are two-to-two orthogonal, and so, all the
eigenfunctions of $\mathcal{L}_R$ corresponding to negative
eigenvalues, which come from $\mathcal{L}_3$ (if they exist) are
orthogonal to $\ker(\mathcal{L}_I)$.
(ii) Since $\mathcal{L}_2$ has no negative eigenvalues, all the
negative eigenvalues of $\mathcal{L}_I$ come from the operator
$\mathcal{L}_3$ (if there exists any negative eigenvalue).
Therefore, the statement follows because $\mathcal{L}_3$ is a
self-adjoint operator. This completes the proof of the theorem.
\end{proof}
\subsection{Orbital Stability} \label{orbital stability}
In this subsection, we establish the stability result for the
periodic travelling waves $(e^{i\gamma t}\phi_\gamma(x),0)$, where
$\phi_\gamma$ is a dnoidal wave given by Theorem \ref{theorem3.1}.
To make clear our notion of orbital stability, we note that the
system \eqref{1.1} has phase and translation symmetries, i.e., if
$(u(x,t),w(x,t))$ is a solution of \eqref{1.1} so are
$$
(e^{is}u(x,t),e^{is}w(x,t)) \quad \mbox{and} \quad
(u(x+r,t),w(x+r,t)),
$$
for any $r,s \in \mathbb{R}$ (we denote these symmetries by $T_p(s)$
and $T_{tr}(r)$, respectively). Therefore, by orbital stability we
mean stability modulo phase and space translation. More precisely.
\begin{definition} \label{definition5.1} \rm
Let $X_1=H_{\rm per}^1([0,L])\times H_{\rm per}^1([0,L])$.
A travelling-wave
solution for \eqref{1.1}, $\Phi(x,t)=(e^{i\gamma t}\phi_\gamma(x),
e^{i\gamma t}\psi_\gamma(x))$, is said to be orbitally stable in
$X_1$ (or $X_1$-stable) if for every $\varepsilon>0$ there exists a
$\delta>0$ such that if $z_0 \in X_1$ and $\|z_0-(\phi_\gamma,
\psi_\gamma) \|_{X_1}< \delta$, then the solution $z(t)=(u(t),w(t))$
of \eqref{1.1} with $z(0)=z_0$ exists for all $t$ and satisfies
\[
\sup_{t\in{\mathbb{R}}} \inf_{s,r \in {\mathbb{R}}} \|
z(t)-T_p(s)T_{tr}(r)(\phi_\gamma,\psi_\gamma) \|_{X_1} <
\varepsilon.
\]
Otherwise, we say that $\Phi(x,t)$ is orbitally unstable in $X_1$
(or $X_1$-unstable).
\end{definition}
Our stability result is as follows.
\begin{theorem} \label{theorem4.6}
Let $\gamma \in \big( \frac{2\pi^2}{L^2}, \infty \big)$, and
assume that $\sigma, \eta>0$ satisfy $\eta>\sigma$. Then, for
$\phi_{\gamma}$ given by Theorem $\ref{theorem3.1}$, the periodic
travelling waves $\Phi_\gamma(x,t)=(e^{i\gamma t}\phi_{\gamma}(x),0)$
are orbitally stable in $X_1$.
\end{theorem}
\begin{proof}
The idea is to apply the theory developed by Grillakis, Shatah, and
Strauss \cite{Grillakis4} for abstract Hamiltonian system. To do so,
we first note that from Corollary \ref{corollary4.3}, the real
Hilbert space $X_{\mathbb{R}}:=[H_{\rm per}^1([0,L])]^4$ can be
orthogonally decomposed as
\[
X_{{\mathbb{R}}}=\mathcal{N} \oplus \ker(\mathcal{L}_\gamma) \oplus \mathcal{P},
\]
where $\mathcal{N}$ denotes the negative eigenspace of
$\mathcal{L}_{\gamma}$ and $\mathcal{P}$ is a closed subspace such
that $\langle\mathcal{L}_{\gamma}p,p \rangle \geq \vartheta_0
\|p\|_{X_1}^2$, for all $p \in \mathcal{P}$ and some
$\vartheta_0>0$.
Next, for $\gamma \in I=\big( \frac{2\pi^2}{L^2}, \infty \big)$
and $\Phi_{\gamma}=(\phi_{\gamma},0,0,0)$, we define the real
function
\begin{equation} \label{functiond}
d(\gamma)=\mathcal{H}(\Phi_{\gamma})+ \gamma
\mathcal{F}(\Phi_{\gamma}),
\end{equation}
Hence, since $\Phi_{\gamma}$ is a critical point of the functional
$\mathcal{H}+ \gamma \mathcal{F}$, $\ker(\mathcal{L}_\gamma)$ is
two-dimensional, and $\mathcal{N}$ is one-dimensional, from the
abstract Stability Theorem in \cite{Grillakis4}, we just need to
prove that $d''(\gamma)>0$. To this end, from \eqref{functiond}, we
immediately obtain
\begin{equation} \label{4.9}
d'(\gamma)= \mathcal{F}(\Phi_{\gamma})=\frac{1}{2} \|\phi_{\gamma}
\|_{L_{\rm per}^2([0,L])}^2.
\end{equation}
But, from the explicit form of $\phi_{\gamma}$, we calculate
\begin{equation} \label{4.10}
\|\phi_{\gamma} \|_{L_{\rm per}^2([0,L])}^2=\frac{8}{L}K(k)
\int_0^{K(k)} dn^2(x;k)dx=\frac{8}{L}K(k)E(k),
\end{equation}
where in the last equality we have used that $\int_0^{K(k)}
dn^2(x;k)dx=E(k)$ (see \cite[pg. 10]{Byrd}, here $E(k)$ denotes the
complete elliptic integral of the second kind). So, from \eqref{4.9}
and \eqref{4.10}, we deduce that
$$
d''(\gamma)=\frac{4}{L}\frac{d}{dk}[K(k)E(k)]\frac{dk}{d\omega}
\frac{1}{\eta}.
$$
Since the mapping $k\in (0,1) \mapsto K(k)E(k)$ is a strictly
increasing function, and because from Theorem \ref{theorem3.1} we
have $\frac{dk}{d\omega}>0$, the theorem follows.
\end{proof}
\subsection*{Remark}
Note that our stability result includes the
case of two waves of different polarizations and the case for two
waves of different frequencies $\omega_1$ and $\omega_2$ with
$\omega_2 \ll \omega_1$ (see the Introduction).
\subsection{Orbital Instability} \label{subsection4.3}
In Subsection \ref{orbital stability}, we established a orbital
stability result for the periodic waves
$\Phi_\gamma(x,t)=(e^{i\gamma t}\phi_{\gamma}(x),0)$, where
$\phi_\gamma$ is given by Theorem \ref{theorem3.1}, by periodic
perturbations which have the same fundamental period of
$\phi_\gamma$. In this subsection, we ask ourselves if such waves
are stable/unstable when we consider periodic perturbations which
have twice the fundamental period of $\phi_\gamma$. As we will see
below, in this case the waves $\Phi_\gamma(x,t)$ are orbitally
unstable but in a weaker sense than in Definition
\ref{definition5.1}. Actually, here our notion of orbital stability
is slightly different from that in Subsection \ref{orbital
stability} and does not include space translations.
\begin{definition} \label{definition6.1} \rm
Let $X_2=H_{\rm per}^1([0,2L])\times H_{\rm per}^1([0,2L])$. The
orbit generated modulo phase, $\{T_p(\gamma
s)(\phi_\gamma,\psi_\gamma); s \in {\mathbb{R}} \}$, is said to be
orbitally stable in $X_2$ (or $X_2$-stable) if for every
$\varepsilon>0$ there exists a $\delta>0$ such that if $z_0 \in X_2$
and $\|z_0-(\phi_\gamma, \psi_\gamma) \|_{X_2}< \delta$, then the
solution $z(t)=(u(t),w(t))$ of \eqref{1.1} with $z(0)=z_0$ exists
for all $t$ and sa\-tisfies
$$
{ \sup_{t\in{\mathbb{R}}} \inf_{s \in {\mathbb{R}}}
\| z(t)-T_p(s)(\phi_\gamma,\psi_\gamma) \|_{X_2} < \varepsilon.}
$$
Otherwise, the orbit is said to be orbitally unstable in $X_2$ (or
$X_2$-unstable).
\end{definition}
Here, we follow the approach introduced by Grillakis in
\cite{Grillakis1}, \cite{Grillakis2} (see also \cite{Angulo2},
\cite{Angulo5}), which get orbital instability from the linear
instability of the zero solution for the linearization of
\eqref{1.4} around the orbit $\{T_p(\gamma
s)(\phi_\gamma,\psi_\gamma); s \in {\mathbb{R}} \}$.
To start, we define the orbit $\mathcal{O}$ to be
$$
\mathcal{O}=\{T_p(\gamma s)(\phi_{\gamma},0,0,0); s \in
{\mathbb{R}} \},
$$
where $\phi_\gamma=\phi$ is the dnoidal wave given
by Theorem \ref{theorem3.1}. Next, for $\Phi=(\phi_{\gamma},0,0,0)$
and $U=(P,R,Q,S)$ (recall that we have written $u=P+iQ$, $w=R+iS$),
we define $V=V(t)$ to be
\[
V=T_p(-\gamma t)U-\Phi.
\]
By using the group properties of $T_p(s)$ together with the fact
that $\Phi$ is a critical point of the functional
$\mathcal{H}+\gamma \mathcal{F}$, it is easy to see, from the Taylor
expansion and \eqref{Hamiltonian}, that $V(t)$ satisfies
\begin{equation} \label{4.12}
\frac{dV}{dt} = J\mathcal{L}_{\gamma}V+O(\|V\|^2),
\end{equation}
where $J$ and $\mathcal{L}_\gamma$ are the operators defined in
\eqref{4.1} and \eqref{4.3}, respectively.
To prove that the linearized equation \eqref{4.12} has zero as an
unstable solution, it is well known that it suffices to show that
$J\mathcal{L}_\gamma$ has one and finitely many eigenvalues with
strictly positive real part. Moreover, this implies that the orbit
$\mathcal{O}$ is orbitally unstable (see \cite{Grillakis1},
\cite{Grillakis4}, \cite{Shatah}). Keeping this in mind, we prove
the following result.
\begin{theorem} \label{theorem4.8}
Let $\gamma \in \big( \frac{2\pi^2}{L^2}, \infty \big)$ and
$\sigma, \eta>0$. Then, for $\phi=\phi_{\gamma}$ given by Theorem
\ref{theorem3.1}, the orbit
$$
\mathcal{O}= \{T_p(\gamma t)(\phi,0,0,0); t \in {\mathbb{R}}\}
$$
is $X_2$-unstable by the periodic flow of the system $(\ref{1.4})$.
\end{theorem}
\begin{proof}
As we have already pointed out, we only have to prove that the
operator $J\mathcal{L}_{\gamma}$ has one and finitely many
eigenvalues with strictly positive real part. But, from Lemma 5.6
and Theorem 5.8 in \cite{Grillakis4}, we know that
$J\mathcal{L}_{\gamma}$ has finitely many eigenvalues with strictly
positive real part. Now, to prove that $J\mathcal{L}_{\gamma}$ has
at least one eigenvalue with strictly positive real part, in light
of Theorem 2.6 in \cite{Grillakis2}, we define
\begin{gather*}
Y= [\ker(\mathcal{L}_{R})\cup \ker(\mathcal{L}_{I})]^{\bot},\\
\widehat{\mathcal{L}}_{R}= \mbox{restriction of }
\mathcal{L}_{R} \mbox{ on } Y\cap H_{\rm per}^2([0,2L]) ,\\
\widehat{\mathcal{L}}_{I}^{-1}= \mbox{restriction of }
\mathcal{L}_{I}^{-1} \mbox{ on } Y\cap H_{\rm per}^2([0,2L]).
\end{gather*}
With these notation, Theorem 2.6 in \cite{Grillakis2} states that
$J\mathcal{L}_\gamma$ has exactly
\begin{equation} \label{4.13}
\max\{ n(\widehat{\mathcal{L}}_{R}),
n(\widehat{\mathcal{L}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))
\end{equation}
$\pm$ pairs of real eigenvalues, where $C(\mathcal{L})=\{ y \in Y;
\langle \mathcal{L}y, y\rangle <0 \}$ denotes the negative cone of
the operator $\mathcal{L}$ and $ d(C(\mathcal{L}))$ denotes the
dimension of a maximal linear subspace that is contained in
$C(\mathcal{L})$.
Therefore, we have proved the theorem if we show that the number in
\eqref{4.13} is strictly positive. Since $\mathcal{L}_{R}$ is a
self-adjoint operator on $L^2_{\rm per}([0,2L])$, its negative
eigenspace is orthogonal to its kernel (the same conclusion holds
for the operator $\mathcal{L}_{I}$). Thus, from Theorem
\ref{theorem4.4}, we have $ n(\widehat{\mathcal{L}}_{R})=k_0+2$ and
$n(\widehat{\mathcal{L}}_{I})=k_0$. Moreover, from the structure of
the operators $\mathcal{L}_{R}$ and $ \mathcal{L}_{I}$, we see that
the negative cone $C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1})$ is $k_0$-dimensional; that is,
$$
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))=k_0.
$$
Hence,
$$
\max\{ n(\widehat{\mathcal{L}}_{R}),
n(\widehat{\mathcal{L}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{L}}_{R}) \cap
C(\widehat{\mathcal{L}}_{I}^{-1}))=k_0+2-k_0=2.
$$
This completes the proof of the theorem.
\end{proof}
\section{Orbital Instability of the non-semitrivial solutions}
\label{section5}
This section is mainly devoted to prove the instability results
concerning the dnoidal-wave solutions given in Theorem
\ref{theorem3.2}. For the sake of simplicity, throughout this
section we take $\eta=1$. So, the coupled system \eqref{1.4} admits
the solutions in Theorem \ref{theorem3.2} for $\sigma \neq 1$ and
$b^2=1$. We assume throughout that $b=1$, but a similar analysis can
be performed if $b=-1$.
\subsection{Spectral Analysis and Stability/Instability}
Fix $L>0$ and let $\widetilde{\Phi}=(\phi_\gamma,\phi_\gamma,0,0)$,
where $\phi=\phi_\gamma$ is the dnoidal wave given by Theorem
\ref{theorem3.2} (with $\eta=b=1$). Consider the linearized operator
\begin{equation} \label{5.1}
\mathcal{T_{\gamma}}=\mathcal{H}''(\widetilde{\Phi})+\gamma
\mathcal{F}''(\widetilde{\Phi})
=\begin{pmatrix}
\mathcal{T}_{R} & 0 \\
0 & \mathcal{T}_{I}
\end{pmatrix},
\end{equation}
where
\begin{gather} \label{5.2}
\mathcal{T}_{R} = \begin{pmatrix}
-\frac{d^2}{dx^2}+\gamma-\left( 3+\sigma \right) \phi_\gamma^2 &
-2\sigma\phi_\gamma^2 \\
-2\sigma\phi_\gamma^2 & -\frac{d^2}{dx^2}+\gamma-\left( 3+\sigma
\right) \phi_\gamma^2
\end{pmatrix},\\
\label{5.3}
\mathcal{T}_{I} = \begin{pmatrix}
-\frac{d^2}{dx^2}+\gamma-\left( 1+\sigma \right) \phi_\gamma^2 & 0
\\
0 & -\frac{d^2}{dx^2}+\gamma-\left( 1+\sigma \right) \phi_\gamma^2
\end{pmatrix}.
\end{gather}
Our first result is about the study of the spectra of the
operators $\mathcal{T}_{R}$ and $\mathcal{T}_{I}$.
\begin{theorem} \label{theorem5.1}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
\ref{theorem3.2} and assume that $\sigma>0$. Then the following
spectral properties hold.
\begin{itemize}
\item [(i)] If $\sigma>1$ then the operator $\mathcal{T}_{R}$ in (\ref{5.2}) defined
in $L^2_{\rm per}([0,L])$ with domain $H_{\rm per}^2([0,L])$ has exactly one negative
eigenvalue which is simple; zero is a simple eigenvalue. Moreover, the remainder
of the spectrum is constituted by a discrete set of
eigenvalues.
\item [(ii)] If $00$, the Comparison Theorem implies
that $\mathcal{T}_{3}$ has at most one negative eigenvalue and the
second eigenvalue is strictly positive.
Now, note that from \eqref{5.7}, we can also write
\begin{equation}\label{5.8}
\mathcal{T}_{3}=\mathcal{T}_{2}+2(\sigma-1) \phi_\gamma^2.
\end{equation}
Because $\mathcal{T}_{2}$ has no negative eigenvalues, it follows
from \eqref{5.8}, the above observation, and the Comparison Theorem
that $\mathcal{T}_{3}$ is a strictly positive operator if
$\sigma>1$, and it has a unique negative eigenvalue if $00$ and if (ii) occurs then $u_1>0$.
Hence, for any $\overrightarrow{\phi}=(\alpha_1 \phi_\gamma,
\alpha_2 \phi_\gamma) \in \ker(\mathcal{T}_{I})$ with $\alpha_1 \neq
\alpha_2$, we have $\langle \overrightarrow{u},
\overrightarrow{\phi} \rangle_{L^2_{\rm per}} \neq 0$. In consequence,
$\overrightarrow{u} \notin [\ker(\mathcal{T}_{I})]^\perp$. This
completes the proof of the theorem.
\end{proof}
In analogy to Theorem \ref{theorem4.8}, we can also prove that the
orbit $\widetilde{\mathcal{O}}$ is orbitally unstable with regard
to periodic perturbations which have twice the fundamental period of
$\phi_\gamma$.
\begin{theorem} \label{theorem5.4}
Let $\phi=\phi_{\gamma}$ be the dnoidal wave given by Theorem
$\ref{theorem3.2}$. Then the orbit
$$
\widetilde{\mathcal{O}}=\{T_p(\gamma
s)(\phi_{\gamma},\phi_\gamma,0,0); s \in {\mathbb{R}} \}
$$
is $X_2$-unstable, in the sense of Definition \ref{definition6.1},
by the periodic flow of system $(\ref{1.4})$.
\end{theorem}
\begin{proof}
The proof follows the same arguments as in Theorem \ref{theorem4.8}.
Actually, similarly to the proof of Theorem \ref{theorem4.4}, we can
prove that the operator $\mathcal{T}_{1}$, with domain
$H^2_{\rm per}([0,2L])$ has exactly three negative eigenvalues, for
which, the eigenfunctions corresponding to the second and third
eigenvalues are orthogonal to $\phi_\gamma$. Therefore,
\[
\max\{ n(\widehat{\mathcal{T}}_{R}),
n(\widehat{\mathcal{T}}_{I}^{-1}) \} -
d(C(\widehat{\mathcal{T}}_{R}) \cap
C(\widehat{\mathcal{T}}_{I}^{-1}))= n(\widehat{\mathcal{T}}_{R})
\geq 2,
\]
and the theorem is proved.
\end{proof}
Finally, we observe that the solutions in Theorem \ref{theorem3.2}
also make sense when $-1