summaryrefslogtreecommitdiff
path: root/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch
diff options
context:
space:
mode:
Diffstat (limited to 'gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch')
-rw-r--r--gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch79
1 files changed, 79 insertions, 0 deletions
diff --git a/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch b/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch
new file mode 100644
index 00000000000..2d58cbf6482
--- /dev/null
+++ b/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch
@@ -0,0 +1,79 @@
1From cebc07fed0707457487656836e627a154416d252 Mon Sep 17 00:00:00 2001
2From: Palash Lalwani <palashlalwani.r@gmail.com>
3Date: Wed, 22 Jul 2026 21:24:52 +0530
4Subject: [PATCH] fix: don't pass grouping columns to
5 groupby().parallel_apply() on pandas >= 3.0
6
7Since pandas 3.0, DataFrameGroupBy.apply operates on _obj_with_exclusions, so the
8grouping columns are no longer passed to the applied function. Iterating a
9DataFrameGroupBy still yields them, which is how pandarallel builds its chunks, so
10parallel_apply kept passing them.
11
12This produced a silent divergence rather than an error: user functions that iterate
13columns or aggregate across the frame got different results from parallel_apply than
14from apply.
15
16Restrict each group to the columns apply would have used, gated on the pandas version
17so behaviour on pandas < 3.0 is unchanged.
18
19The groupby test fixture read df.b, but the test also covers groupby(["a", "b"]) where
20b is now a grouping column, making the reference apply() call raise on pandas 3.0.
21Switched the fixture to column c, which is never a grouping column in these tests.
22---
23 pandarallel/data_types/dataframe_groupby.py | 18 +++++++++++++++++-
24 tests/test_pandarallel.py | 8 ++++++--
25 2 files changed, 23 insertions(+), 3 deletions(-)
26
27diff --git a/pandarallel/data_types/dataframe_groupby.py b/pandarallel/data_types/dataframe_groupby.py
28index f2dbb91..9b448d7 100644
29--- a/pandarallel/data_types/dataframe_groupby.py
30+++ b/pandarallel/data_types/dataframe_groupby.py
31@@ -17,8 +17,24 @@ def get_chunks(
32 chunks = chunk(dataframe_groupby.ngroups, nb_workers)
33 iterator = iter(dataframe_groupby)
34
35+ # Since pandas 3.0, `DataFrameGroupBy.apply` operates on
36+ # `_obj_with_exclusions`, i.e. the grouping columns are no longer passed to
37+ # the applied function. Iterating a `DataFrameGroupBy` still yields them
38+ # though, so restrict each group to the columns `apply` would have used.
39+ # On pandas < 3.0 the grouping columns are still passed, so keep them.
40+ columns = (
41+ dataframe_groupby._obj_with_exclusions.columns
42+ if get_pandas_version() >= (3, 0)
43+ else None
44+ )
45+
46 for chunk_ in chunks:
47- yield [next(iterator) for _ in range(chunk_.stop - chunk_.start)]
48+ groups = [next(iterator) for _ in range(chunk_.stop - chunk_.start)]
49+
50+ if columns is not None:
51+ groups = [(key, df[columns]) for key, df in groups]
52+
53+ yield groups
54
55 @staticmethod
56 def work(
57diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py
58index 0f91c32..2f43458 100644
59--- a/tests/test_pandarallel.py
60+++ b/tests/test_pandarallel.py
61@@ -93,12 +93,16 @@ def func(x):
62
63 @pytest.fixture()
64 def func_dataframe_groupby_apply():
65+ # Uses column `c`, which is never a grouping column in the tests below. Since pandas
66+ # 3.0 the grouping columns are not passed to the applied function, so a function
67+ # reading `df.b` would fail on the `groupby(["a", "b"])` case before `parallel_apply`
68+ # is even reached.
69 def func(df):
70 dum = 0
71- for item in df.b:
72+ for item in df.c:
73 dum += math.log10(math.sqrt(math.exp(item**2)))
74
75- return dum / len(df.b)
76+ return dum / len(df.c)
77
78 return func
79