Using any awk:

 $ cat tst.awk
 match($0, /\\SomeStyle\{/) {
 head = substr($0,1,RSTART-1)
 tail = substr($0,RSTART+RLENGTH-1)
 
 gsub(/@/, "@A", tail)
 gsub(/</, "@B", tail)
 gsub(/>/, "@C", tail)
 
 while ( match(tail, /{[^{}]*}/) ) {
 if ( RSTART == 1 ) {
 break
 }
 tail = substr(tail,1,RSTART-1) "<" substr(tail,RSTART+1,RLENGTH-2) ">" substr(tail,RSTART+RLENGTH)
 }
 
 gsub( /</, "{", tail)
 gsub( />/, "}", tail)
 gsub(/@C/, ">", tail)
 gsub(/@B/, "<", tail)
 gsub(/@A/, "@", tail)
 
 $0 = head tail
 }
 { print }

<p>

 $ awk -f tst.awk file
 {\otherstyle{this is the \textit{nested part} some more text...}}

The above doesn't attempt to handle escaped `{` or `}` in the input because it'd need to treat `\{` (escaped `{`) differently from `\\{` (escaped ```\``` followed by `{`) and that requires more thought than I'm willing to put into this given it doesn't appear in the sample input and so probably isn't actually an issue for the OP and they can always ask a followup question if it is and they don't already have a way to handle it.

Here's a commented version of the above in case that's useful.

 match($0, /\\SomeStyle\{/) {
 # Save the part before \SomeStyle{ as-is in head so we can add it back later
 head = substr($0,1,RSTART-1)
 # Save the part starting from the { in \SomeStyle{ in tail for further processing
 tail = substr($0,RSTART+RLENGTH-1)
 
 # Convert every @ to @A so we can then convert every < to @B and > to @C
 # so we can later replace every { in the nested string to < and > to }
 # so on every loop iteration we get rid of the innermost matched { and }
 gsub(/@/, "@A", tail)
 gsub(/</, "@B", tail)
 gsub(/>/, "@C", tail)
 
 # Loop finding every innermost {...} substring, replacing the { and } with
 # < and > as we go to catch longer and longer nested substrings on each
 # iteration, stopping when RSTART is 1 because thats the start of the
 # outermost {...}. Add "print tail" after "tail = ..." to see what it is
 # doing if its not obvious.
 while ( match(tail, /{[^{}]*}/) ) {
 if ( RSTART == 1 ) {
 # Weve found the outermost {...} so stop looping
 break
 }
 # Change {...{foo}...} to {...<foo>...}
 tail = substr(tail,1,RSTART-1) "<" substr(tail,RSTART+1,RLENGTH-2) ">" substr(tail,RSTART+RLENGTH)
 }
 
 # Now change all replacement strings back to their original characters
 gsub( /</, "{", tail)
 gsub( />/, "}", tail)
 gsub(/@C/, ">", tail)
 gsub(/@B/, "<", tail)
 gsub(/@A/, "@", tail)
 
 # Reconstruct $0 from the part before \SomeStyle{ and the processed part
 $0 = head tail
 }
 { print }