For anyone in the future looking for a solution to this using Unity/C#, here is an implementation I made based on Steve's answer:
//takes a rotation and returns the rotation that is the closest with all axes pointing at 90 degree intervals to the identity quaternion Quaternion snapToNearestRightAngle(Quaternion currentRotation) { Vector3 closestToForward = closestToAxis(currentRotation, Vector3.forward); Vector3 closestToUp = closestToAxis(currentRotation, Vector3.up); return Quaternion.LookRotation(closestToForward, closestToUp); } //finds the axis that is closest to the currentRotations local axis Vector3 closestToAxis(Quaternion currentRotation, Vector3 axis) { Vector3[] checkAxes = new Vector3[] { Vector3.forward, Vector3.right, Vector3.up, -Vector3.forward, -Vector3.right, -Vector3.up }; Vector3 closestToAxis = Vector3.forward; float highestDot = -1; foreach (Vector3 checkAxis in checkAxes) check(ref highestDot, ref closestToAxis, currentRotation, axis, checkAxis); return closestToAxis; } //finds the closest axis to the input rotations specified axis void check(ref float highestDot, ref Vector3 closest, Quaternion currentRotation, Vector3 axis, Vector3 checkDir) { float dot = Vector3.Dot(currentRotation * axis, checkDir); if (dot > highestDot) { highestDot = dot; closest = checkDir; } }
And here is an implementation based on this post which is much more efficient:
//takes a rotation and returns the rotation that is the closest with all axes pointing at 90 degree intervals to the identity quaternion Quaternion snapToNearestRightAngle(Quaternion currentRotation) { Vector3 closestToForward = SnappedToNearestAxis(currentRotation * Vector3.forward); Vector3 closestToUp = SnappedToNearestAxis(currentRotation * Vector3.up); return Quaternion.LookRotation(closestToForward, closestToUp); } //find the world axis that is closest to direction Vector3 SnappedToNearestAxis(Vector3 direction) { float x = Mathf.Abs(direction.x); float y = Mathf.Abs(direction.y); float z = Mathf.Abs(direction.z); if (x > y && x > z) { return new Vector3(Mathf.Sign(direction.x), 0, 0); } else if (y > x && y > z) { return new Vector3(0, Mathf.Sign(direction.y), 0); } else { return new Vector3(0, 0, Mathf.Sign(direction.z)); } }