Basically I have an orthographic camera that is looking at a simple square. The camera is sitting above the square and tilted by x: 45° and y: 45°, so the camera sees the square as a rhombus / diamond shape.
I want the camera to be moveable within a rectangular boundary that appears to be leveled with the camera, so it would need to be rotated by 45° around the y-axis (up-vector in Unity) in World-coordinates. I already figured out the math to constrain the camera to this shape, but there is a weird thing going on when the movement gets clipped: The camera doesn't follow the outline perfectly, but in a rounded shape (see screenshot). Could this be due to a rounding error? Code snipped below. 
using System; using UnityEngine; using static UnityEngine.Mathf; public class InputController : MonoBehaviour { private const float CamHeight = 10f; [SerializeField] private GameObject gameBoard; [SerializeField] private GameObject camBounds; [SerializeField] private float camBoundRadius = 5f; private Vector3 _camOrigin; private MeshCollider _collider; private Vector3 _difference; // Change in position of mouse relative to origin private Camera _mainCamera; private Vector3 _origin; // Place where mouse is first pressed private Plane _worldPlane; private void Start() { _mainCamera = Camera.main; if (_mainCamera is null) throw new Exception("No main camera could be found!"); // Set the camera so that the board is always in the center _camOrigin = new Vector3( -CamHeight / Sqrt(2f), CamHeight, -CamHeight / Sqrt(2f)); _mainCamera.transform.position = _camOrigin; _collider = gameBoard.GetComponent<MeshCollider>(); _worldPlane = new Plane(Vector3.up, 0); // For dragging the camera around } private void LateUpdate() { if (Input.GetMouseButtonDown(0)) _origin = MouseWorldPosition(); if (Input.GetMouseButton(0)) { _difference = MouseWorldPosition() - transform.position; var newPos = _origin - _difference; var absDiff = newPos - _camOrigin; var angle = Angle2Pi(Vector3.right, absDiff); var constraint = SawTooth(angle); if (absDiff.magnitude < constraint) transform.position = newPos; else transform.position = _camOrigin + absDiff.normalized * constraint; } } private Vector3 MouseWorldPosition() { var clickedPoint = Input.mousePosition; var ray = _mainCamera.ScreenPointToRay(clickedPoint); var worldPos = Vector3.zero; if (_worldPlane.Raycast(ray, out var raycastHit)) worldPos = ray.GetPoint(raycastHit); return worldPos; } private float SawTooth(float theta) { var x = 4f / PI * camBoundRadius * (1f - 1f / Sqrt(2f)); x *= Abs(theta % (PI / 2f) - PI / 4f); x += camBoundRadius / Sqrt(2f); return x; } private static float Angle2Pi(Vector3 from, Vector3 to, Vector3 normal = default) { if (normal == default) normal = Vector3.up; var angle = Vector3.Angle(from, to) * (PI / 180f); var sign = Sign(Vector3.Dot(normal, Vector3.Cross(from, to))); angle = 2f * PI - angle * sign; return angle; } } 