iterparse.pxi

# iterparse -- event-driven parsing

DEF __ITERPARSE_CHUNK_SIZE = 32768

cdef class iterparse:
    u"""iterparse(self, source, events=("end",), tag=None, \
                  attribute_defaults=False, dtd_validation=False, \
                  load_dtd=False, no_network=True, remove_blank_text=False, \
                  remove_comments=False, remove_pis=False, encoding=None, \
                  html=False, recover=None, huge_tree=False, schema=None)

    Incremental parser.

    Parses XML into a tree and generates tuples (event, element) in a
    SAX-like fashion. ``event`` is any of 'start', 'end', 'start-ns',
    'end-ns'.

    For 'start' and 'end', ``element`` is the Element that the parser just
    found opening or closing.  For 'start-ns', it is a tuple (prefix, URI) of
    a new namespace declaration.  For 'end-ns', it is simply None.  Note that
    all start and end events are guaranteed to be properly nested.

    The keyword argument ``events`` specifies a sequence of event type names
    that should be generated.  By default, only 'end' events will be
    generated.

    The additional ``tag`` argument restricts the 'start' and 'end' events to
    those elements that match the given tag.  The ``tag`` argument can also be
    a sequence of tags to allow matching more than one tag.  By default,
    events are generated for all elements.  Note that the 'start-ns' and
    'end-ns' events are not impacted by this restriction.

    The other keyword arguments in the constructor are mainly based on the
    libxml2 parser configuration.  A DTD will also be loaded if validation or
    attribute default values are requested.

    Available boolean keyword arguments:
     - attribute_defaults: read default attributes from DTD
     - dtd_validation: validate (if DTD is available)
     - load_dtd: use DTD for parsing
     - no_network: prevent network access for related files
     - remove_blank_text: discard blank text nodes
     - remove_comments: discard comments
     - remove_pis: discard processing instructions
     - strip_cdata: replace CDATA sections by normal text content (default: True)
     - compact: safe memory for short text content (default: True)
     - resolve_entities: replace entities by their text value (default: True)
     - huge_tree: disable security restrictions and support very deep trees
                  and very long text content (only affects libxml2 2.7+)
     - html: parse input as HTML (default: XML)
     - recover: try hard to parse through broken input (default: True for HTML,
                False otherwise)

    Other keyword arguments:
     - encoding: override the document encoding
     - schema: an XMLSchema to validate against
    """
    cdef _FeedParser _parser
    cdef object _tag
    cdef object _events
    cdef readonly object root
    cdef object _source
    cdef object _filename
    cdef object _error
    cdef bint _close_source_after_read

    def __init__(self, source, events=(u"end",), *, tag=None,
                 attribute_defaults=False, dtd_validation=False,
                 load_dtd=False, no_network=True, remove_blank_text=False,
                 compact=True, resolve_entities=True, remove_comments=False,
                 remove_pis=False, strip_cdata=True, encoding=None,
                 html=False, recover=None, huge_tree=False, collect_ids=True,
                 XMLSchema schema=None):
        if not hasattr(source, 'read'):
            source = _getFSPathOrObject(source)
            self._filename = source
            if python.IS_PYTHON2:
                source = _encodeFilename(source)
            source = open(source, 'rb')
            self._close_source_after_read = True
        else:
            self._filename = _getFilenameForFile(source)
            self._close_source_after_read = False

        if recover is None:
            recover = html

        if html:
            # make sure we're not looking for namespaces
            events = [event for event in events
                      if event not in ('start-ns', 'end-ns')]
            parser = HTMLPullParser(
                events,
                tag=tag,
                recover=recover,
                base_url=self._filename,
                encoding=encoding,
                remove_blank_text=remove_blank_text,
                remove_comments=remove_comments,
                remove_pis=remove_pis,
                strip_cdata=strip_cdata,
                no_network=no_network,
                target=None,  # TODO
                schema=schema,
                compact=compact)
        else:
            parser = XMLPullParser(
                events,
                tag=tag,
                recover=recover,
                base_url=self._filename,
                encoding=encoding,
                attribute_defaults=attribute_defaults,
                dtd_validation=dtd_validation,
                load_dtd=load_dtd,
                no_network=no_network,
                schema=schema,
                huge_tree=huge_tree,
                remove_blank_text=remove_blank_text,
                resolve_entities=resolve_entities,
                remove_comments=remove_comments,
                remove_pis=remove_pis,
                strip_cdata=strip_cdata,
                collect_ids=True,
                target=None,  # TODO
                compact=compact)

        self._events = parser.read_events()
        self._parser = parser
        self._source = source

    @property
    def error_log(self):
        """The error log of the last (or current) parser run.
        """
        return self._parser.feed_error_log

    @property
    def resolvers(self):
        """The custom resolver registry of the last (or current) parser run.
        """
        return self._parser.resolvers

    @property
    def version(self):
        """The version of the underlying XML parser."""
        return self._parser.version

    def set_element_class_lookup(self, ElementClassLookup lookup = None):
        u"""set_element_class_lookup(self, lookup = None)

        Set a lookup scheme for element classes generated from this parser.

        Reset it by passing None or nothing.
        """
        self._parser.set_element_class_lookup(lookup)

    def makeelement(self, _tag, attrib=None, nsmap=None, **_extra):
        u"""makeelement(self, _tag, attrib=None, nsmap=None, **_extra)

        Creates a new element associated with this parser.
        """
        self._parser.makeelement(
            _tag, attrib=None, nsmap=None, **_extra)

    @cython.final
    cdef _close_source(self):
        if self._source is None:
            return
        if not self._close_source_after_read:
            self._source = None
            return
        try:
            close = self._source.close
        except AttributeError:
            close = None
        finally:
            self._source = None
        if close is not None:
            close()

    def __iter__(self):
        return self

    def __next__(self):
        try:
            return next(self._events)
        except StopIteration:
            pass
        context = <_SaxParserContext>self._parser._getPushParserContext()
        if self._source is not None:
            done = False
            while not done:
                try:
                    done = self._read_more_events(context)
                    return next(self._events)
                except StopIteration:
                    pass  # no events yet
                except Exception as e:
                    self._error = e
                    self._close_source()
                    try:
                        return next(self._events)
                    except StopIteration:
                        break
        # nothing left to read or return
        if self._error is not None:
            error = self._error
            self._error = None
            raise error
        if (context._validator is not None
                and not context._validator.isvalid()):
            _raiseParseError(context._c_ctxt, self._filename,
                             context._error_log)
        # no errors => all done
        raise StopIteration

    @cython.final
    cdef bint _read_more_events(self, _SaxParserContext context) except -123:
        data = self._source.read(__ITERPARSE_CHUNK_SIZE)
        if not isinstance(data, bytes):
            self._close_source()
            raise TypeError("reading file objects must return bytes objects")
        if not data:
            try:
                self.root = self._parser.close()
            finally:
                self._close_source()
            return True
        self._parser.feed(data)
        return False


cdef enum _IterwalkSkipStates:
    IWSKIP_NEXT_IS_START
    IWSKIP_SKIP_NEXT
    IWSKIP_CAN_SKIP
    IWSKIP_CANNOT_SKIP


cdef class iterwalk:
    u"""iterwalk(self, element_or_tree, events=("end",), tag=None)

    A tree walker that generates events from an existing tree as if it
    was parsing XML data with ``iterparse()``.

    Just as for ``iterparse()``, the ``tag`` argument can be a single tag or a
    sequence of tags.

    After receiving a 'start' or 'start-ns' event, the children and
    descendants of the current element can be excluded from iteration
    by calling the ``skip_subtree()`` method.
    """
    cdef _MultiTagMatcher _matcher
    cdef list   _node_stack
    cdef list   _events
    cdef object _pop_event
    cdef object _include_siblings
    cdef int    _index
    cdef int    _event_filter
    cdef _IterwalkSkipStates _skip_state

    def __init__(self, element_or_tree, events=(u"end",), tag=None):
        cdef _Element root
        cdef int ns_count
        root = _rootNodeOrRaise(element_or_tree)
        self._event_filter = _buildParseEventFilter(events)
        if tag is None or tag == '*':
            self._matcher = None
        else:
            self._matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag)
        self._node_stack  = []
        self._events = []
        self._pop_event = self._events.pop
        self._skip_state = IWSKIP_CANNOT_SKIP  # ignore all skip requests by default

        if self._event_filter:
            self._index = 0
            if self._matcher is not None and self._event_filter & PARSE_EVENT_FILTER_START:
                self._matcher.cacheTags(root._doc)

            # When processing an ElementTree, add events for the preceding comments/PIs.
            if self._event_filter & (PARSE_EVENT_FILTER_COMMENT | PARSE_EVENT_FILTER_PI):
                if isinstance(element_or_tree, _ElementTree):
                    self._include_siblings = root
                    for elem in list(root.itersiblings(preceding=True))[::-1]:
                        if self._event_filter & PARSE_EVENT_FILTER_COMMENT and elem.tag is Comment:
                            self._events.append((u'comment', elem))
                        elif self._event_filter & PARSE_EVENT_FILTER_PI and elem.tag is PI:
                            self._events.append((u'pi', elem))

            ns_count = self._start_node(root)
            self._node_stack.append( (root, ns_count) )
        else:
            self._index = -1

    def __iter__(self):
        return self

    def __next__(self):
        cdef xmlNode* c_child
        cdef _Element node
        cdef _Element next_node
        cdef int ns_count = 0
        if self._events:
            return self._next_event()
        if self._matcher is not None and self._index >= 0:
            node = self._node_stack[self._index][0]
            self._matcher.cacheTags(node._doc)

        # find next node
        while self._index >= 0:
            node = self._node_stack[self._index][0]

            if self._skip_state == IWSKIP_SKIP_NEXT:
                c_child = NULL
            else:
                c_child = self._process_non_elements(
                    node._doc, _findChildForwards(node._c_node, 0))
            self._skip_state = IWSKIP_CANNOT_SKIP

            while c_child is NULL:
                # back off through parents
                self._index -= 1
                node = self._end_node()
                if self._index < 0:
                    break
                c_child = self._process_non_elements(
                    node._doc, _nextElement(node._c_node))

            if c_child is not NULL:
                next_node = _elementFactory(node._doc, c_child)
                if self._event_filter & (PARSE_EVENT_FILTER_START |
                                         PARSE_EVENT_FILTER_START_NS):
                    ns_count = self._start_node(next_node)
                elif self._event_filter & PARSE_EVENT_FILTER_END_NS:
                    ns_count = _countNsDefs(next_node._c_node)
                self._node_stack.append( (next_node, ns_count) )
                self._index += 1
            if self._events:
                return self._next_event()

        if self._include_siblings is not None:
            node, self._include_siblings = self._include_siblings, None
            self._process_non_elements(node._doc, _nextElement(node._c_node))
            if self._events:
                return self._next_event()

        raise StopIteration

    @cython.final
    cdef xmlNode* _process_non_elements(self, _Document doc, xmlNode* c_node):
        while c_node is not NULL and c_node.type != tree.XML_ELEMENT_NODE:
            if c_node.type == tree.XML_COMMENT_NODE:
                if self._event_filter & PARSE_EVENT_FILTER_COMMENT:
                    self._events.append(
                        (u"comment", _elementFactory(doc, c_node)))
                c_node = _nextElement(c_node)
            elif c_node.type == tree.XML_PI_NODE:
                if self._event_filter & PARSE_EVENT_FILTER_PI:
                    self._events.append(
                        (u"pi", _elementFactory(doc, c_node)))
                c_node = _nextElement(c_node)
            else:
                break
        return c_node

    @cython.final
    cdef _next_event(self):
        if self._skip_state == IWSKIP_NEXT_IS_START:
            if self._events[0][0] in (u'start', u'start-ns'):
                self._skip_state = IWSKIP_CAN_SKIP
        return self._pop_event(0)

    def skip_subtree(self):
        """Prevent descending into the current subtree.
        Instead, the next returned event will be the 'end' event of the current element
        (if included), ignoring any children or descendants.

        This has no effect right after an 'end' or 'end-ns' event.
        """
        if self._skip_state == IWSKIP_CAN_SKIP:
            self._skip_state = IWSKIP_SKIP_NEXT

    @cython.final
    cdef int _start_node(self, _Element node) except -1:
        cdef int ns_count
        if self._event_filter & PARSE_EVENT_FILTER_START_NS:
            ns_count = _appendStartNsEvents(node._c_node, self._events)
            if self._events:
                self._skip_state = IWSKIP_NEXT_IS_START
        elif self._event_filter & PARSE_EVENT_FILTER_END_NS:
            ns_count = _countNsDefs(node._c_node)
        else:
            ns_count = 0
        if self._event_filter & PARSE_EVENT_FILTER_START:
            if self._matcher is None or self._matcher.matches(node._c_node):
                self._events.append( (u"start", node) )
                self._skip_state = IWSKIP_NEXT_IS_START
        return ns_count

    @cython.final
    cdef _Element _end_node(self):
        cdef _Element node
        cdef int i, ns_count
        node, ns_count = self._node_stack.pop()
        if self._event_filter & PARSE_EVENT_FILTER_END:
            if self._matcher is None or self._matcher.matches(node._c_node):
                self._events.append( (u"end", node) )
        if self._event_filter & PARSE_EVENT_FILTER_END_NS and ns_count:
            event = (u"end-ns", None)
            for i in range(ns_count):
                self._events.append(event)
        return node


cdef int _countNsDefs(xmlNode* c_node):
    cdef xmlNs* c_ns
    cdef int count
    count = 0
    c_ns = c_node.nsDef
    while c_ns is not NULL:
        count += (c_ns.href is not NULL)
        c_ns = c_ns.next
    return count


cdef int _appendStartNsEvents(xmlNode* c_node, list event_list) except -1:
    cdef xmlNs* c_ns
    cdef int count
    count = 0
    c_ns = c_node.nsDef
    while c_ns is not NULL:
        if c_ns.href:
            ns_tuple = (funicodeOrEmpty(c_ns.prefix),
                        funicode(c_ns.href))
            event_list.append( (u"start-ns", ns_tuple) )
            count += 1
        c_ns = c_ns.next
    return count