detect overflow in combinations (closes #23366)

Lib/test/test_itertools.py

@@ -137,6 +137,11 @@ class TestBasicOps(unittest.TestCase):
                 self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version
                 self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version
 
+    @test_support.bigaddrspacetest
+    def test_combinations_overflow(self):
+        with self.assertRaises(OverflowError):
+            combinations("AA", 2**29)
+
     @test_support.impl_detail("tuple reuse is specific to CPython")
     def test_combinations_tuple_reuse(self):
         self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1)

Misc/NEWS

@@ -18,6 +18,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #23366: Fixed possible integer overflow in itertools.combinations.
+
 - Issue #23191: fnmatch functions that use caching are now threadsafe.
 
 - Issue #18518: timeit now rejects statements which can't be compiled outside

Modules/itertoolsmodule.c

@@ -2093,6 +2093,10 @@ combinations_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
         goto error;
     }
 
+    if (r > PY_SSIZE_T_MAX/sizeof(Py_ssize_t)) {
+        PyErr_SetString(PyExc_OverflowError, "r is too big");
+        goto error;
+    }
     indices = PyMem_Malloc(r * sizeof(Py_ssize_t));
     if (indices == NULL) {
         PyErr_NoMemory();