diff options
Diffstat (limited to 'gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch')
| -rw-r--r-- | gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch | 79 |
1 files changed, 79 insertions, 0 deletions
diff --git a/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch b/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch new file mode 100644 index 00000000000..2d58cbf6482 --- /dev/null +++ b/gnu/packages/patches/python-pandarallel-fix-parallel_apply.patch | |||
| @@ -0,0 +1,79 @@ | |||
| 1 | From cebc07fed0707457487656836e627a154416d252 Mon Sep 17 00:00:00 2001 | ||
| 2 | From: Palash Lalwani <palashlalwani.r@gmail.com> | ||
| 3 | Date: Wed, 22 Jul 2026 21:24:52 +0530 | ||
| 4 | Subject: [PATCH] fix: don't pass grouping columns to | ||
| 5 | groupby().parallel_apply() on pandas >= 3.0 | ||
| 6 | |||
| 7 | Since pandas 3.0, DataFrameGroupBy.apply operates on _obj_with_exclusions, so the | ||
| 8 | grouping columns are no longer passed to the applied function. Iterating a | ||
| 9 | DataFrameGroupBy still yields them, which is how pandarallel builds its chunks, so | ||
| 10 | parallel_apply kept passing them. | ||
| 11 | |||
| 12 | This produced a silent divergence rather than an error: user functions that iterate | ||
| 13 | columns or aggregate across the frame got different results from parallel_apply than | ||
| 14 | from apply. | ||
| 15 | |||
| 16 | Restrict each group to the columns apply would have used, gated on the pandas version | ||
| 17 | so behaviour on pandas < 3.0 is unchanged. | ||
| 18 | |||
| 19 | The groupby test fixture read df.b, but the test also covers groupby(["a", "b"]) where | ||
| 20 | b is now a grouping column, making the reference apply() call raise on pandas 3.0. | ||
| 21 | Switched the fixture to column c, which is never a grouping column in these tests. | ||
| 22 | --- | ||
| 23 | pandarallel/data_types/dataframe_groupby.py | 18 +++++++++++++++++- | ||
| 24 | tests/test_pandarallel.py | 8 ++++++-- | ||
| 25 | 2 files changed, 23 insertions(+), 3 deletions(-) | ||
| 26 | |||
| 27 | diff --git a/pandarallel/data_types/dataframe_groupby.py b/pandarallel/data_types/dataframe_groupby.py | ||
| 28 | index f2dbb91..9b448d7 100644 | ||
| 29 | --- a/pandarallel/data_types/dataframe_groupby.py | ||
| 30 | +++ b/pandarallel/data_types/dataframe_groupby.py | ||
| 31 | @@ -17,8 +17,24 @@ def get_chunks( | ||
| 32 | chunks = chunk(dataframe_groupby.ngroups, nb_workers) | ||
| 33 | iterator = iter(dataframe_groupby) | ||
| 34 | |||
| 35 | + # Since pandas 3.0, `DataFrameGroupBy.apply` operates on | ||
| 36 | + # `_obj_with_exclusions`, i.e. the grouping columns are no longer passed to | ||
| 37 | + # the applied function. Iterating a `DataFrameGroupBy` still yields them | ||
| 38 | + # though, so restrict each group to the columns `apply` would have used. | ||
| 39 | + # On pandas < 3.0 the grouping columns are still passed, so keep them. | ||
| 40 | + columns = ( | ||
| 41 | + dataframe_groupby._obj_with_exclusions.columns | ||
| 42 | + if get_pandas_version() >= (3, 0) | ||
| 43 | + else None | ||
| 44 | + ) | ||
| 45 | + | ||
| 46 | for chunk_ in chunks: | ||
| 47 | - yield [next(iterator) for _ in range(chunk_.stop - chunk_.start)] | ||
| 48 | + groups = [next(iterator) for _ in range(chunk_.stop - chunk_.start)] | ||
| 49 | + | ||
| 50 | + if columns is not None: | ||
| 51 | + groups = [(key, df[columns]) for key, df in groups] | ||
| 52 | + | ||
| 53 | + yield groups | ||
| 54 | |||
| 55 | @staticmethod | ||
| 56 | def work( | ||
| 57 | diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py | ||
| 58 | index 0f91c32..2f43458 100644 | ||
| 59 | --- a/tests/test_pandarallel.py | ||
| 60 | +++ b/tests/test_pandarallel.py | ||
| 61 | @@ -93,12 +93,16 @@ def func(x): | ||
| 62 | |||
| 63 | @pytest.fixture() | ||
| 64 | def func_dataframe_groupby_apply(): | ||
| 65 | + # Uses column `c`, which is never a grouping column in the tests below. Since pandas | ||
| 66 | + # 3.0 the grouping columns are not passed to the applied function, so a function | ||
| 67 | + # reading `df.b` would fail on the `groupby(["a", "b"])` case before `parallel_apply` | ||
| 68 | + # is even reached. | ||
| 69 | def func(df): | ||
| 70 | dum = 0 | ||
| 71 | - for item in df.b: | ||
| 72 | + for item in df.c: | ||
| 73 | dum += math.log10(math.sqrt(math.exp(item**2))) | ||
| 74 | |||
| 75 | - return dum / len(df.b) | ||
| 76 | + return dum / len(df.c) | ||
| 77 | |||
| 78 | return func | ||
| 79 | |||
