Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Enabling binary operations with list-like Python objects. #2054

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Enabling binary operations with list-like Python objects. #2054

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
2e20719
Add tests
itholic Feb 15, 2021
922ba6a
Fix rsub
itholic Feb 15, 2021
02c3334
Add Index
itholic Feb 18, 2021
8df4ff9
Fix test for mod and rmod
itholic Feb 18, 2021
5d2d1c5
Use pandas
itholic Feb 18, 2021
0fd3666
Fix test
itholic Feb 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion databricks/koalas/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ def __rsub__(self, other) -> Union["Series", "Index"]:
return -column_op(F.datediff)(self, F.lit(other)).astype("long")
else:
raise TypeError("date subtraction can only be applied to date series.")
return column_op(Column.__rsub__)(self, other)
return column_op(lambda left, right: right - left)(self, other)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI: Column.__rsub__ doesn't support pyspark.sql.column.Column for second parameter.

>>> kdf = ks.DataFrame({"A": [1, 2, 3, 4], "B": [10, 20, 30, 40]})
>>> sdf = kdf.to_spark()
>>> col1 = sdf.A
>>> col2 = sdf.B
>>> Column.__rsub__(col1, col2)
Traceback (most recent call last):
...
TypeError: Column is not iterable

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does support:

>>> Column.__rsub__(df.id, 1)
Column<'(1 - id)'>

It doesn't work in your case above because the instance is Spark column. In practice, that wouldn't happen because it will only be called when the first operand doesn't know how to handle Spark column e.g.) 1 - df.id.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does it cause any exception?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use column_op(Column.__rsub__)(self, other) as it is, it raises TypeError: Column is not iterable for the case below.

>>> kser = ks.Series([1, 2, 3, 4])
>>> [10, 20, 30, 40] - kser
Traceback (most recent call last):
...
TypeError: Column is not iterable

Copy link
Collaborator

@ueshin ueshin Feb 18, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that this case must be handled in lines 490-492. We can move back to Column.__rsub__.


def __rmul__(self, other) -> Union["Series", "Index"]:
if isinstance(other, str):
Expand Down
56 changes: 56 additions & 0 deletions databricks/koalas/series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,6 +452,10 @@ def spark_type(self):
def add(self, other) -> "Series":
return self + other

def __add__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
itholic marked this conversation as resolved.
Show resolved Hide resolved
return IndexOpsMixin.__add__(self, other)

add.__doc__ = _flex_doc_SERIES.format(
desc="Addition",
op_name="+",
Expand All @@ -463,6 +467,10 @@ def add(self, other) -> "Series":
def radd(self, other) -> "Series":
return other + self

def __radd__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__radd__(self, other)

radd.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Addition",
op_name="+",
Expand Down Expand Up @@ -498,6 +506,10 @@ def rdiv(self, other) -> "Series":
def truediv(self, other) -> "Series":
return self / other

def __truediv__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__truediv__(self, other)

truediv.__doc__ = _flex_doc_SERIES.format(
desc="Floating division",
op_name="/",
Expand All @@ -509,6 +521,10 @@ def truediv(self, other) -> "Series":
def rtruediv(self, other) -> "Series":
return other / self

def __rtruediv__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rtruediv__(self, other)

rtruediv.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Floating division",
op_name="/",
Expand All @@ -520,6 +536,10 @@ def rtruediv(self, other) -> "Series":
def mul(self, other) -> "Series":
return self * other

def __mul__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__mul__(self, other)

mul.__doc__ = _flex_doc_SERIES.format(
desc="Multiplication",
op_name="*",
Expand All @@ -533,6 +553,10 @@ def mul(self, other) -> "Series":
def rmul(self, other) -> "Series":
return other * self

def __rmul__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rmul__(self, other)

rmul.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Multiplication",
op_name="*",
Expand All @@ -544,6 +568,10 @@ def rmul(self, other) -> "Series":
def sub(self, other) -> "Series":
return self - other

def __sub__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__sub__(self, other)

sub.__doc__ = _flex_doc_SERIES.format(
desc="Subtraction",
op_name="-",
Expand All @@ -557,6 +585,10 @@ def sub(self, other) -> "Series":
def rsub(self, other) -> "Series":
return other - self

def __rsub__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rsub__(self, other)

rsub.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Subtraction",
op_name="-",
Expand All @@ -568,6 +600,10 @@ def rsub(self, other) -> "Series":
def mod(self, other) -> "Series":
return self % other

def __mod__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__mod__(self, other)

mod.__doc__ = _flex_doc_SERIES.format(
desc="Modulo",
op_name="%",
Expand All @@ -579,6 +615,10 @@ def mod(self, other) -> "Series":
def rmod(self, other) -> "Series":
return other % self

def __rmod__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rmod__(self, other)

rmod.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Modulo",
op_name="%",
Expand All @@ -590,6 +630,10 @@ def rmod(self, other) -> "Series":
def pow(self, other) -> "Series":
return self ** other

def __pow__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__pow__(self, other)

pow.__doc__ = _flex_doc_SERIES.format(
desc="Exponential power of series",
op_name="**",
Expand All @@ -601,6 +645,10 @@ def pow(self, other) -> "Series":
def rpow(self, other) -> "Series":
return other ** self

def __rpow__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rpow__(self, other)

rpow.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Exponential power",
op_name="**",
Expand All @@ -612,6 +660,10 @@ def rpow(self, other) -> "Series":
def floordiv(self, other) -> "Series":
return self // other

def __floordiv__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__floordiv__(self, other)

floordiv.__doc__ = _flex_doc_SERIES.format(
desc="Integer division",
op_name="//",
Expand All @@ -623,6 +675,10 @@ def floordiv(self, other) -> "Series":
def rfloordiv(self, other) -> "Series":
return other // self

def __rfloordiv__(self, other):
other = ks.Index(other, name=self.name) if isinstance(other, (list, tuple)) else other
return IndexOpsMixin.__rfloordiv__(self, other)

rfloordiv.__doc__ = _flex_doc_SERIES.format(
desc="Reverse Integer division",
op_name="//",
Expand Down