fix traverse_nested when value is not compatible with ItemAdapter (#450)

scrapinghub · May 16, 2024 · 2d11fbf · 2d11fbf
1 parent 6ea9f67
commit 2d11fbf
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 2 deletions.
diff --git a/spidermon/contrib/utils/attributes.py b/spidermon/contrib/utils/attributes.py
@@ -1,4 +1,5 @@
 from typing import Any, List
+
 from itemadapter import ItemAdapter
 
 
@@ -14,7 +15,9 @@ def traverse_nested(obj: ItemAdapter, keys: List[str]) -> ItemAdapter:
             # Traverse next level of item object
             key = keys.pop(0)
             current_obj = ItemAdapter(current_obj[key])
-        except KeyError:
+        # KeyError: Key does not exist
+        # TypeError: Key is not compatible with ItemAdapter (None or unsupported type)
+        except (KeyError, TypeError):
             raise KeyError(f'Invalid key "{key}" for {current_obj} in {obj}')
 
     return current_obj

diff --git a/tests/contrib/utils/test_attributes.py b/tests/contrib/utils/test_attributes.py
@@ -1,7 +1,8 @@
-import pytest
 from dataclasses import dataclass
 
+import pytest
 from itemadapter import ItemAdapter
+
 from spidermon.contrib.utils.attributes import (
     get_nested_attribute,
     set_nested_attribute,
@@ -19,6 +20,11 @@ def test_get_nested_attribute():
     with pytest.raises(KeyError):
         get_nested_attribute(item, "attr1.missing_attribute.attr2")
 
+    # Intermiddle attribute is None
+    item = ItemAdapter({"foo": None})
+    with pytest.raises(KeyError):
+        get_nested_attribute(item, "foo.missing_attribute")
+
 
 def test_set_nested_attribute():
     item = ItemAdapter({"foo": None, "attr1": {"attr2": {"attr3": None}}})