In order to avoid using explicit distances, I'd use the following code.
\documentclass[12pt,a4paper]{article} % CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068 % Load TikZ \usepackage{tikz} \usetikzlibrary{matrix,decorations.pathreplacing,calc} % Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit \pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}} \pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}} \pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}} \newcommand\mymatrixbraceoffseth{0.5em} \newcommand\mymatrixbraceoffsetv{0.2em} \newcommand*\mymatrixbraceright[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$) -- node[left=2pt] {#4} ($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbraceleft[4][m]{ \draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$) -- node[right=2pt] {#4} ($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbracetop[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$) -- node[above=2pt] {#4} ($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$); } \newcommand*\mymatrixbracebottom[4][m]{ \draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$) -- node[below=2pt] {#4} ($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$); } % CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068 \tikzset{ cheating dash/.code args={on #1 off #2}{ % Use csname so catcode of @ doesn't have do be changed. \csname tikz@addoption\endcsname{% \pgfgetpath\currentpath% \pgfprocessround{\currentpath}{\currentpath}% \csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}% \pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult% \pgfmathparse{#1+#2}\let\onoff=\pgfmathresult% \pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult% \pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult% \pgfsetdash{{#1}{\offexpand}}{0pt}}% } } \begin{document} \begin{equation} \mathbf{X} = \begin{tikzpicture}[baseline=0cm,mymatrixenv] \matrix [mymatrix,inner sep=4pt,row sep=1em] (m) { \frac{x}{y} + C & m\omega^2 + U \\ 0 & 1 \\ 1 & \frac{x}{y} + Dx^2 \\ }; \path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X) (m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y); % Dashed lines \draw [cheating dash=on 2pt off 2pt,red] (X |- m.north) -- (X |- m.south); \draw [cheating dash=on 2pt off 2pt,red] (Y -| m.west) -- (Y -| m.east); % Braces \mymatrixbraceright{1}{3}{$A$} \mymatrixbracetop{2}{2}{$B$} \end{tikzpicture} \end{equation} \end{document}

EDIT: Using your own suggestion (with some minor modifications), the code can be simplified to:
\documentclass[12pt,a4paper]{article} % CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068 % Load TikZ \usepackage{tikz} \usetikzlibrary{matrix,decorations.pathreplacing,calc} % Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit \pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}} \pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}} \pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}} \newcommand\mymatrixbraceoffseth{0.5em} \newcommand\mymatrixbraceoffsetv{0.2em} \newcommand*\mymatrixbraceright[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$) -- node[left=2pt] {#4} ($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbraceleft[4][m]{ \draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$) -- node[right=2pt] {#4} ($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbracetop[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$) -- node[above=2pt] {#4} ($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$); } \newcommand*\mymatrixbracebottom[4][m]{ \draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$) -- node[below=2pt] {#4} ($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$); } % CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068 \tikzset{ cheating dash/.code args={on #1 off #2}{ % Use csname so catcode of @ doesn't have do be changed. \csname tikz@addoption\endcsname{% \pgfgetpath\currentpath% \pgfprocessround{\currentpath}{\currentpath}% \csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}% \pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult% \pgfmathparse{#1+#2}\let\onoff=\pgfmathresult% \pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult% \pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult% \pgfsetdash{{#1}{\offexpand}}{0pt}}% } } \newcommand*\mymatrixdashedlineH[4][]{ \path (#2-#3-1.south) -- (#2-#4-1.north) coordinate [midway] (Y); \draw [#1] (Y -| #2.west) -- (Y -| #2.east);} \newcommand*\mymatrixdashedlineV[4][]{ \path (#2-1-#3.east) -- (#2-1-#4.west) coordinate [midway] (X); \draw [#1] (X |- #2.north) -- (X |- #2.south);} \begin{document} \begin{equation} \mathbf{X} = \begin{tikzpicture}[baseline=0cm,mymatrixenv] \matrix [mymatrix,inner sep=4pt,row sep=1em] (m) { \frac{x}{y} + C & m\omega^2 + U \\ 0 & 1 \\ 1 & \frac{x}{y} + Dx^2 \\ }; \path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X) (m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y); % Dashed lines \mymatrixdashedlineV[red,dashed]{m}{1}{2} \mymatrixdashedlineH[red,dashed]{m}{1}{2} % Braces \mymatrixbraceright{1}{3}{$A$} \mymatrixbracetop{2}{2}{$B$} \end{tikzpicture} \end{equation} \end{document}
EDIT: Just for fun: some TikZy versions of your nice proposals. (NOTE: I am not saying that this is a safe path since I am using \pgfextra. It is really just for fun.)
\documentclass[12pt,a4paper]{article} % CODE BELOW FROM https://tex.stackexchange.com/a/1070/128068 % Load TikZ \usepackage{tikz} \usetikzlibrary{matrix,decorations.pathreplacing,calc} % Set various styles for the matrices and braces. It might pay off to fiddle around with the values a little bit \pgfkeys{tikz/mymatrixenv/.style={decoration=brace,every left delimiter/.style={xshift=3pt},every right delimiter/.style={xshift=-3pt}}} \pgfkeys{tikz/mymatrix/.style={matrix of math nodes,left delimiter=[,right delimiter={]},inner sep=2pt,column sep=1em,row sep=0.5em,nodes={inner sep=0pt}}} \pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}} \newcommand\mymatrixbraceoffseth{0.5em} \newcommand\mymatrixbraceoffsetv{0.2em} \newcommand*\mymatrixbraceright[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-#3-1.south west)!(#1.south west)-(\mymatrixbraceoffseth,0)$) -- node[left=2pt] {#4} ($(#1.north west)!(#1-#2-1.north west)!(#1.south west)-(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbraceleft[4][m]{ \draw[mymatrixbrace] ($(#1.north east)!(#1-#2-1.north east)!(#1.south east)+(\mymatrixbraceoffseth,0)$) -- node[right=2pt] {#4} ($(#1.north east)!(#1-#3-1.south east)!(#1.south east)+(\mymatrixbraceoffseth,0)$); } \newcommand*\mymatrixbracetop[4][m]{ \draw[mymatrixbrace] ($(#1.north west)!(#1-1-#2.north west)!(#1.north east)+(0,\mymatrixbraceoffsetv)$) -- node[above=2pt] {#4} ($(#1.north west)!(#1-1-#3.north east)!(#1.north east)+(0,\mymatrixbraceoffsetv)$); } \newcommand*\mymatrixbracebottom[4][m]{ \draw[mymatrixbrace] ($(#1.south west)!(#1-1-#3.south east)!(#1.south east)-(0,\mymatrixbraceoffsetv)$) -- node[below=2pt] {#4} ($(#1.south west)!(#1-1-#2.south west)!(#1.south east)-(0,\mymatrixbraceoffsetv)$); } % CHEATING DASH FROM https://tex.stackexchange.com/a/133357/128068 \tikzset{ cheating dash/.code args={on #1 off #2}{ % Use csname so catcode of @ doesn't have do be changed. \csname tikz@addoption\endcsname{% \pgfgetpath\currentpath% \pgfprocessround{\currentpath}{\currentpath}% \csname pgf@decorate@parsesoftpath\endcsname{\currentpath}{\currentpath}% \pgfmathparse{\csname pgf@decorate@totalpathlength\endcsname-#1}\let\rest=\pgfmathresult% \pgfmathparse{#1+#2}\let\onoff=\pgfmathresult% \pgfmathparse{max(floor(\rest/\onoff), 1)}\let\nfullonoff=\pgfmathresult% \pgfmathparse{max((\rest-\onoff*\nfullonoff)/\nfullonoff+#2, #2)}\let\offexpand=\pgfmathresult% \pgfsetdash{{#1}{\offexpand}}{0pt}}% } } \tikzset{ vertical line/.style args={with style #1 right of column #2 in row #3}{ append after command={ \pgfextra{ \pgfmathparse{int(#2+1)} \path (\tikzlastnode-#3-#2.east) -- (\tikzlastnode-#3-\pgfmathresult.west) coordinate [midway] (MiloX); \draw [#1] (MiloX |- \tikzlastnode.north) -- (MiloX |- \tikzlastnode.south); } } }, horizontal line/.style args={with style #1 below row #2 in column #3}{append after command={\pgfextra{ \pgfmathparse{int(#2+1)} \path (\tikzlastnode-#2-#3.south) -- (\tikzlastnode-\pgfmathresult-#3.north) coordinate [midway] (MiloY); \draw [#1] (MiloY -| \tikzlastnode.west) -- (MiloY -| \tikzlastnode.east); }} } } \begin{document} \begin{equation} \mathbf{X} = \begin{tikzpicture}[baseline=0cm,mymatrixenv] \matrix [mymatrix,inner sep=4pt,row sep=1em, vertical line={with style {dashed,red} right of column 1 in row 1}, horizontal line={with style {dashed,red} below row 2 in column 1}] (m) { \frac{x}{y} + C & m\omega^2 + U \\ 0 & 1 \\ 1 & \frac{x}{y} + Dx^2 \\ }; \path (m-1-1.east) -- (m-1-2.west) coordinate[midway] (X) (m-1-1.south) -- (m-2-1.north) coordinate[midway] (Y); % Braces \mymatrixbraceright{1}{3}{$A$} \mymatrixbracetop{2}{2}{$B$} \end{tikzpicture} \end{equation} \end{document}