Fix ConvexHull to return points in counter-clockwise order

- Add sortCounterClockwise method to ensure CCW ordering
- Start from bottom-most, left-most point for deterministic results
- Fix issue where unordered HashSet broke downstream algorithms
- Add comprehensive tests with CCW order verification

src/main/java/com/thealgorithms/geometry/ConvexHull.java

@@ -61,11 +61,24 @@ public static List<Point> convexHullBruteForce(List<Point> points) {
         return new ArrayList<>(convexSet);
     }
 
+    /**
+     * Computes the convex hull using a recursive divide-and-conquer approach.
+     * Returns points in counter-clockwise order starting from the bottom-most, left-most point.
+     *
+     * @param points the input points
+     * @return the convex hull points in counter-clockwise order
+     */
     public static List<Point> convexHullRecursive(List<Point> points) {
+        if (points.size() < 3) {
+            List<Point> result = new ArrayList<>(points);
+            Collections.sort(result);
+            return result;
+        }
+
         Collections.sort(points);
         Set<Point> convexSet = new HashSet<>();
-        Point leftMostPoint = points.get(0);
-        Point rightMostPoint = points.get(points.size() - 1);
+        Point leftMostPoint = points.getFirst();
+        Point rightMostPoint = points.getLast();
 
         convexSet.add(leftMostPoint);
         convexSet.add(rightMostPoint);
@@ -85,9 +98,8 @@ public static List<Point> convexHullRecursive(List<Point> points) {
         constructHull(upperHull, leftMostPoint, rightMostPoint, convexSet);
         constructHull(lowerHull, rightMostPoint, leftMostPoint, convexSet);
 
-        List<Point> result = new ArrayList<>(convexSet);
-        Collections.sort(result);
-        return result;
+        // Convert to list and sort in counter-clockwise order
+        return sortCounterClockwise(new ArrayList<>(convexSet));
     }
 
     private static void constructHull(Collection<Point> points, Point left, Point right, Set<Point> convexSet) {
@@ -114,4 +126,62 @@ private static void constructHull(Collection<Point> points, Point left, Point ri
             }
         }
     }
+
+    /**
+     * Sorts convex hull points in counter-clockwise order starting from
+     * the bottom-most, left-most point.
+     *
+     * @param hullPoints the unsorted convex hull points
+     * @return the points sorted in counter-clockwise order
+     */
+    private static List<Point> sortCounterClockwise(List<Point> hullPoints) {
+        if (hullPoints.size() <= 2) {
+            Collections.sort(hullPoints);
+            return hullPoints;
+        }
+
+        // Find the bottom-most, left-most point (pivot)
+        Point pivot = hullPoints.getFirst();
+        for (Point p : hullPoints) {
+            if (p.y() < pivot.y() || (p.y() == pivot.y() && p.x() < pivot.x())) {
+                pivot = p;
+            }
+        }
+
+        // Sort other points by polar angle with respect to pivot
+        final Point finalPivot = pivot;
+        List<Point> sorted = new ArrayList<>(hullPoints);
+        sorted.remove(finalPivot);
+
+        sorted.sort((p1, p2) -> {
+            int crossProduct = Point.orientation(finalPivot, p1, p2);
+
+            if (crossProduct == 0) {
+                // Collinear points: sort by distance from pivot (closer first for convex hull)
+                long dist1 = distanceSquared(finalPivot, p1);
+                long dist2 = distanceSquared(finalPivot, p2);
+                return Long.compare(dist1, dist2);
+            }
+
+            // Positive cross product means p2 is counter-clockwise from p1
+            // We want counter-clockwise order, so if p2 is CCW from p1, p1 should come first
+            return -crossProduct;
+        });
+
+        // Build result with pivot first
+        List<Point> result = new ArrayList<>();
+        result.add(finalPivot);
+        result.addAll(sorted);
+
+        return result;
+    }
+
+    /**
+     * Computes the squared distance between two points to avoid floating point operations.
+     */
+    private static long distanceSquared(Point p1, Point p2) {
+        long dx = (long) p1.x() - p2.x();
+        long dy = (long) p1.y() - p2.y();
+        return dx * dx + dy * dy;
+    }
 }

src/test/java/com/thealgorithms/geometry/ConvexHullTest.java

@@ -1,6 +1,7 @@
 package com.thealgorithms.geometry;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.Arrays;
 import java.util.List;
@@ -10,31 +11,117 @@ public class ConvexHullTest {
 
     @Test
     void testConvexHullBruteForce() {
+        // Test 1: Triangle with intermediate point
         List<Point> points = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 1));
         List<Point> expected = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 1));
         assertEquals(expected, ConvexHull.convexHullBruteForce(points));
 
+        // Test 2: Collinear points
         points = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 0));
         expected = Arrays.asList(new Point(0, 0), new Point(10, 0));
         assertEquals(expected, ConvexHull.convexHullBruteForce(points));
 
+        // Test 3: Complex polygon
         points = Arrays.asList(new Point(0, 3), new Point(2, 2), new Point(1, 1), new Point(2, 1), new Point(3, 0), new Point(0, 0), new Point(3, 3), new Point(2, -1), new Point(2, -4), new Point(1, -3));
         expected = Arrays.asList(new Point(2, -4), new Point(1, -3), new Point(0, 0), new Point(3, 0), new Point(0, 3), new Point(3, 3));
         assertEquals(expected, ConvexHull.convexHullBruteForce(points));
     }
 
     @Test
     void testConvexHullRecursive() {
+        // Test 1: Triangle - CCW order starting from bottom-left
+        // The algorithm includes (1,0) as it's detected as an extreme point
         List<Point> points = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 1));
+        List<Point> result = ConvexHull.convexHullRecursive(points);
         List<Point> expected = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 1));
-        assertEquals(expected, ConvexHull.convexHullRecursive(points));
+        assertEquals(expected, result);
+        assertTrue(isCounterClockwise(result), "Points should be in counter-clockwise order");
 
+        // Test 2: Collinear points
         points = Arrays.asList(new Point(0, 0), new Point(1, 0), new Point(10, 0));
+        result = ConvexHull.convexHullRecursive(points);
         expected = Arrays.asList(new Point(0, 0), new Point(10, 0));
-        assertEquals(expected, ConvexHull.convexHullRecursive(points));
+        assertEquals(expected, result);
 
-        points = Arrays.asList(new Point(0, 3), new Point(2, 2), new Point(1, 1), new Point(2, 1), new Point(3, 0), new Point(0, 0), new Point(3, 3), new Point(2, -1), new Point(2, -4), new Point(1, -3));
-        expected = Arrays.asList(new Point(2, -4), new Point(1, -3), new Point(0, 0), new Point(3, 0), new Point(0, 3), new Point(3, 3));
-        assertEquals(expected, ConvexHull.convexHullRecursive(points));
+        // Test 3: Complex polygon
+        // Convex hull vertices in CCW order from bottom-most point (2,-4):
+        // (2,-4) -> (3,0) -> (3,3) -> (0,3) -> (0,0) -> (1,-3) -> back to (2,-4)
+        points = Arrays.asList(
+                new Point(0, 3), new Point(2, 2), new Point(1, 1),
+                new Point(2, 1), new Point(3, 0), new Point(0, 0),
+                new Point(3, 3), new Point(2, -1), new Point(2, -4),
+                new Point(1, -3)
+        );
+        result = ConvexHull.convexHullRecursive(points);
+        expected = Arrays.asList(
+                new Point(2, -4),  // Bottom-most, left-most (starting point)
+                new Point(3, 0),   // Right side going up
+                new Point(3, 3),   // Top right corner
+                new Point(0, 3),   // Top left corner
+                new Point(0, 0),   // Left side coming down
+                new Point(1, -3)   // Bottom section, back towards start
+        );
+        assertEquals(expected, result);
+        assertTrue(isCounterClockwise(result), "Points should be in counter-clockwise order");
+    }
+
+    @Test
+    void testConvexHullRecursiveAdditionalCases() {
+        // Test 4: Square (all corners on hull)
+        List<Point> points = Arrays.asList(
+                new Point(0, 0), new Point(2, 0),
+                new Point(2, 2), new Point(0, 2)
+        );
+        List<Point> result = ConvexHull.convexHullRecursive(points);
+        List<Point> expected = Arrays.asList(
+                new Point(0, 0), new Point(2, 0),
+                new Point(2, 2), new Point(0, 2)
+        );
+        assertEquals(expected, result);
+        assertTrue(isCounterClockwise(result), "Square points should be in CCW order");
+
+        // Test 5: Pentagon with interior point
+        points = Arrays.asList(
+                new Point(0, 0), new Point(4, 0), new Point(5, 3),
+                new Point(2, 5), new Point(-1, 3), new Point(2, 2) // (2,2) is interior
+        );
+        result = ConvexHull.convexHullRecursive(points);
+        // CCW from (0,0): (0,0) -> (4,0) -> (5,3) -> (2,5) -> (-1,3)
+        expected = Arrays.asList(
+                new Point(0, 0), new Point(4, 0), new Point(5, 3),
+                new Point(2, 5), new Point(-1, 3)
+        );
+        assertEquals(expected, result);
+        assertTrue(isCounterClockwise(result), "Pentagon points should be in CCW order");
+
+        // Test 6: Simple triangle (clearly convex)
+        points = Arrays.asList(
+                new Point(0, 0), new Point(4, 0), new Point(2, 3)
+        );
+        result = ConvexHull.convexHullRecursive(points);
+        expected = Arrays.asList(
+                new Point(0, 0), new Point(4, 0), new Point(2, 3)
+        );
+        assertEquals(expected, result);
+        assertTrue(isCounterClockwise(result), "Triangle points should be in CCW order");
+    }
+
+    /**
+     * Helper method to verify if points are in counter-clockwise order.
+     * Uses the signed area method: positive area means CCW.
+     */
+    private boolean isCounterClockwise(List<Point> points) {
+        if (points.size() < 3) {
+            return true; // Less than 3 points, trivially true
+        }
+
+        long signedArea = 0;
+        for (int i = 0; i < points.size(); i++) {
+            Point p1 = points.get(i);
+            Point p2 = points.get((i + 1) % points.size());
+            signedArea += (long) p1.x() * p2.y() - (long) p2.x() * p1.y();
+        }
+
+        return signedArea > 0; // Positive signed area means counter-clockwise
     }
 }