C (gcc), 180 168 159 157 bytes

f(char*s){int j=strlen(s),m=j--/2,p=-1,i=0;for(;p&&i<m;j--,i++)s[i]^s[j]&&(!++p&&(s[i+1]==s[j]&&(p=++i)||s[i]==s[j-1]&&(p=j--+1)));return*s&&p<0?m+1:p?p:-1;} 

Try it online!

2 bytes shaved off thanks to ceilingcat!

Ungolfed:

f(char *s) { int j = strlen(s), // j = length of input m = j-- / 2, // m = midpoint of string, // j = index of right character p = -1, // p = -1 means no extra character found yet, 0 means invalid input i = 0; // i = index of left character for (; p && i < m; j--, i++) // loop over the string from both sides, // as long as input is valid. // increment i and decrement j. s[i] ^ s[j] && ( // if (left character != right character) { !++p && ( // if (we didn't remove a character yet) { s[i + 1] == s[j] && ( // if (left+1 char == right char) p = ++i // remove left char, ) || s[i] == s[j - 1] && ( // else if (left char == right-1 char) p = j-- + 1 // remove right char, ) ) // } else the input is invalid. ); // } return *s && p < 0 ? // if (non-zero string and we didn't remove a character yet) m + 1 // return the midpoint character, : p ? // else if (we did remove a character) p // return that character, : -1; // else the input was invalid. } ```