Package lxml :: Package html |
[hide private]
[frames] | no frames]

# Source Code for Package lxml.html

       1  # Copyright (c) 2004 Ian Bicking. All rights reserved.
       2  #
       3  # Redistribution and use in source and binary forms, with or without
       4  # modification, are permitted provided that the following conditions are
       5  # met:
       6  #
       7  # 1. Redistributions of source code must retain the above copyright
       8  # notice, this list of conditions and the following disclaimer.
       9  #
      10  # 2. Redistributions in binary form must reproduce the above copyright
      11  # notice, this list of conditions and the following disclaimer in
      12  # the documentation and/or other materials provided with the
      13  # distribution.
      14  #
      15  # 3. Neither the name of Ian Bicking nor the names of its contributors may
      16  # be used to endorse or promote products derived from this software
      17  # without specific prior written permission.
      18  #
      19  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
      20  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
      21  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
      22  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR
      23  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
      24  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
      25  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
      26  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
      27  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
      28  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      29  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30
      31  """The ``lxml.html`` tool set for HTML handling.
      32  """
      33
      34  from __future__ import absolute_import
      35
      36  __all__ = [
      37      'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring',
      38      'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form',
      39      'find_rel_links', 'find_class', 'make_links_absolute',
      40      'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser', 'parse']
      41
      42
      43  import copy
      44  import sys
      45  import re
      46  from functools import partial
      47
      48  try:
      49      from collections.abc import MutableMapping, MutableSet
      50  except ImportError:
      51      from collections import MutableMapping, MutableSet
      52
      53  from .. import etree
      54  from . import defs
      55  from ._setmixin import SetMixin
      56
      57  try:
      58      from urlparse import urljoin
      59  except ImportError:
      60      # Python 3
      61      from urllib.parse import urljoin
      62
      63  try:
      64      unicode
      65  except NameError:
      66      # Python 3
      67      unicode = str
      68  try:
      69      basestring
      70  except NameError:
      71      # Python 3
      72      basestring = (str, bytes)
      73
      74
      75 -def __fix_docstring(s):
      76      if not s:
      77          return s
      78      if sys.version_info[0] >= 3:
      79          sub = re.compile(r"^(\s*)u'", re.M).sub
      80      else:
      81          sub = re.compile(r"^(\s*)b'", re.M).sub
      82      return sub(r"\1'", s)
      83
      84
      85  XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
      86
      87  _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]",
      88                                 namespaces={'x':XHTML_NAMESPACE})
      89  _options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option",
      90                               namespaces={'x':XHTML_NAMESPACE})
      91  _forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form",
      92                             namespaces={'x':XHTML_NAMESPACE})
      93  #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'})
      94  _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]")
      95  _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]")
      96  _collect_string_content = etree.XPath("string()")
      97  _iter_css_urls = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I).finditer
      98  _iter_css_imports = re.compile(r'@import "(.*?)"').finditer
      99  _label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]",
     100                             namespaces={'x':XHTML_NAMESPACE})
     101  _archive_re = re.compile(r'[^ ]+')
     102  _parse_meta_refresh_url = re.compile(
     103      r'[^;=]*;\s*(?:url\s*=\s*)?(?P<url>.*)$', re.I).search
     104
     105
     106 -def _unquote_match(s, pos):
     107      if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'":
     108          return s[1:-1], pos+1
     109      else:
     110          return s,pos
     111
     112
     113 -def _transform_result(typ, result):
     114      """Convert the result back into the input type.
     115      """
     116      if issubclass(typ, bytes):
     117          return tostring(result, encoding='utf-8')
     118      elif issubclass(typ, unicode):
     119          return tostring(result, encoding='unicode')
     120      else:
     121          return result
     122
     123
     124 -def _nons(tag):
     125      if isinstance(tag, basestring):
     126          if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
     127              return tag.split('}')[-1]
     128      return tag
     129
     130
     131 -class Classes(MutableSet):
     132      """Provides access to an element's class attribute as a set-like collection.
     133      Usage::
     134
     135          >>> el = fromstring('<p class="hidden large">Text</p>')
     136          >>> classes = el.classes  # or: classes = Classes(el.attrib)
     137          >>> classes |= ['block', 'paragraph']
     138          >>> el.get('class')
     139          'hidden large block paragraph'
     140          >>> classes.toggle('hidden')
     141          False
     142          >>> el.get('class')
     143          'large block paragraph'
     144          >>> classes -= ('some', 'classes', 'block')
     145          >>> el.get('class')
     146          'large paragraph'
     147      """
     148 -    def __init__(self, attributes):
     149          self._attributes = attributes
     150          self._get_class_value = partial(attributes.get, 'class', '')
     151
     152 -    def add(self, value):
     153          """
     154          Add a class.
     155
     156          This has no effect if the class is already present.
     157          """
     158          if not value or re.search(r'\s', value):
     159              raise ValueError("Invalid class name: %r" % value)
     160          classes = self._get_class_value().split()
     161          if value in classes:
     162              return
     163          classes.append(value)
     164          self._attributes['class'] = ' '.join(classes)
     165
     166 -    def discard(self, value):
     167          """
     168          Remove a class if it is currently present.
     169
     170          If the class is not present, do nothing.
     171          """
     172          if not value or re.search(r'\s', value):
     173              raise ValueError("Invalid class name: %r" % value)
     174          classes = [name for name in self._get_class_value().split()
     175                     if name != value]
     176          if classes:
     177              self._attributes['class'] = ' '.join(classes)
     178          elif 'class' in self._attributes:
     179              del self._attributes['class']
     180
     181 -    def remove(self, value):
     182          """
     183          Remove a class; it must currently be present.
     184
     185          If the class is not present, raise a KeyError.
     186          """
     187          if not value or re.search(r'\s', value):
     188              raise ValueError("Invalid class name: %r" % value)
     189          super(Classes, self).remove(value)
     190
     191 -    def __contains__(self, name):
     192          classes = self._get_class_value()
     193          return name in classes and name in classes.split()
     194
     195 -    def __iter__(self):
     196          return iter(self._get_class_value().split())
     197
     198 -    def __len__(self):
     199          return len(self._get_class_value().split())
     200
     201      # non-standard methods
     202
     203 -    def update(self, values):
     204          """
     205          Add all names from 'values'.
     206          """
     207          classes = self._get_class_value().split()
     208          extended = False
     209          for value in values:
     210              if value not in classes:
     211                  classes.append(value)
     212                  extended = True
     213          if extended:
     214              self._attributes['class'] = ' '.join(classes)
     215
     216 -    def toggle(self, value):
     217          """
     218          Add a class name if it isn't there yet, or remove it if it exists.
     219
     220          Returns true if the class was added (and is now enabled) and
     221          false if it was removed (and is now disabled).
     222          """
     223          if not value or re.search(r'\s', value):
     224              raise ValueError("Invalid class name: %r" % value)
     225          classes = self._get_class_value().split()
     226          try:
     227              classes.remove(value)
     228              enabled = False
     229          except ValueError:
     230              classes.append(value)
     231              enabled = True
     232          if classes:
     233              self._attributes['class'] = ' '.join(classes)
     234          else:
     235              del self._attributes['class']
     236          return enabled
     237
     238
     239 -class HtmlMixin(object):
     240
     241 -    def set(self, key, value=None):
     242          """set(self, key, value=None)
     243
     244          Sets an element attribute.  If no value is provided, or if the value is None,
     245          creates a 'boolean' attribute without value, e.g. "<form novalidate></form>"
     246          for ``form.set('novalidate')``.
     247          """
     248          super(HtmlElement, self).set(key, value)
     249
     250      @property
     251 -    def classes(self):
     252          """
     253          A set-like wrapper around the 'class' attribute.
     254          """
     255          return Classes(self.attrib)
     256
     257      @classes.setter
     258 -    def classes(self, classes):
     259          assert isinstance(classes, Classes)  # only allow "el.classes |= ..." etc.
     260          value = classes._get_class_value()
     261          if value:
     262              self.set('class', value)
     263          elif self.get('class') is not None:
     264              del self.attrib['class']
     265
     266      @property
     267 -    def base_url(self):
     268          """
     269          Returns the base URL, given when the page was parsed.
     270
     271          Use with ``urlparse.urljoin(el.base_url, href)`` to get
     272          absolute URLs.
     273          """
     274          return self.getroottree().docinfo.URL
     275
     276      @property
     277 -    def forms(self):
     278          """
     279          Return a list of all the forms
     280          """
     281          return _forms_xpath(self)
     282
     283      @property
     284 -    def body(self):
     285          """
     286          Return the <body> element.  Can be called from a child element
     287          to get the document's head.
     288          """
     289          return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0]
     290
     291      @property
     292 -    def head(self):
     293          """
     294          Returns the <head> element.  Can be called from a child
     295          element to get the document's head.
     296          """
     297          return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0]
     298
     299      @property
     300 -    def label(self):
     301          """
     302          Get or set any <label> element associated with this element.
     303          """
     304          id = self.get('id')
     305          if not id:
     306              return None
     307          result = _label_xpath(self, id=id)
     308          if not result:
     309              return None
     310          else:
     311              return result[0]
     312
     313      @label.setter
     314 -    def label(self, label):
     315          id = self.get('id')
     316          if not id:
     317              raise TypeError(
     318                  "You cannot set a label for an element (%r) that has no id"
     319                  % self)
     320          if _nons(label.tag) != 'label':
     321              raise TypeError(
     322                  "You can only assign label to a label element (not %r)"
     323                  % label)
     324          label.set('for', id)
     325
     326      @label.deleter
     327 -    def label(self):
     328          label = self.label
     329          if label is not None:
     330              del label.attrib['for']
     331
     332 -    def drop_tree(self):
     333          """
     334          Removes this element from the tree, including its children and
     335          text.  The tail text is joined to the previous element or
     336          parent.
     337          """
     338          parent = self.getparent()
     339          assert parent is not None
     340          if self.tail:
     341              previous = self.getprevious()
     342              if previous is None:
     343                  parent.text = (parent.text or '') + self.tail
     344              else:
     345                  previous.tail = (previous.tail or '') + self.tail
     346          parent.remove(self)
     347
     348 -    def drop_tag(self):
     349          """
     350          Remove the tag, but not its children or text.  The children and text
     351          are merged into the parent.
     352
     353          Example::
     354
     355              >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>')
     356              >>> h.find('.//b').drop_tag()
     357              >>> print(tostring(h, encoding='unicode'))
     358              <div>Hello World!</div>
     359          """
     360          parent = self.getparent()
     361          assert parent is not None
     362          previous = self.getprevious()
     363          if self.text and isinstance(self.tag, basestring):
     364              # not a Comment, etc.
     365              if previous is None:
     366                  parent.text = (parent.text or '') + self.text
     367              else:
     368                  previous.tail = (previous.tail or '') + self.text
     369          if self.tail:
     370              if len(self):
     371                  last = self[-1]
     372                  last.tail = (last.tail or '') + self.tail
     373              elif previous is None:
     374                  parent.text = (parent.text or '') + self.tail
     375              else:
     376                  previous.tail = (previous.tail or '') + self.tail
     377          index = parent.index(self)
     378          parent[index:index+1] = self[:]
     379
     380 -    def find_rel_links(self, rel):
     381          """
     382          Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements.
     383          """
     384          rel = rel.lower()
     385          return [el for el in _rel_links_xpath(self)
     386                  if el.get('rel').lower() == rel]
     387
     388 -    def find_class(self, class_name):
     389          """
     390          Find any elements with the given class name.
     391          """
     392          return _class_xpath(self, class_name=class_name)
     393
     394 -    def get_element_by_id(self, id, *default):
     395          """
     396          Get the first element in a document with the given id.  If none is
     397          found, return the default argument if provided or raise KeyError
     398          otherwise.
     399
     400          Note that there can be more than one element with the same id,
     401          and this isn't uncommon in HTML documents found in the wild.
     402          Browsers return only the first match, and this function does
     403          the same.
     404          """
     405          try:
     406              # FIXME: should this check for multiple matches?
     407              # browsers just return the first one
     408              return _id_xpath(self, id=id)[0]
     409          except IndexError:
     410              if default:
     411                  return default[0]
     412              else:
     413                  raise KeyError(id)
     414
     415 -    def text_content(self):
     416          """
     417          Return the text content of the tag (and the text in any children).
     418          """
     419          return _collect_string_content(self)
     420
     421 -    def cssselect(self, expr, translator='html'):
     422          """
     423          Run the CSS expression on this element and its children,
     424          returning a list of the results.
     425
     426          Equivalent to lxml.cssselect.CSSSelect(expr, translator='html')(self)
     427          -- note that pre-compiling the expression can provide a substantial
     428          speedup.
     429          """
     430          # Do the import here to make the dependency optional.
     431          from lxml.cssselect import CSSSelector
     432          return CSSSelector(expr, translator=translator)(self)
     433
     434      ########################################
     435      ## Link functions
     436      ########################################
     437
     438 -    def make_links_absolute(self, base_url=None, resolve_base_href=True,
     439                              handle_failures=None):
     440          """
     441          Make all links in the document absolute, given the
     442          ``base_url`` for the document (the full URL where the document
     443          came from), or if no ``base_url`` is given, then the ``.base_url``
     444          of the document.
     445
     446          If ``resolve_base_href`` is true, then any ``<base href>``
     447          tags in the document are used *and* removed from the document.
     448          If it is false then any such tag is ignored.
     449
     450          If ``handle_failures`` is None (default), a failure to process
     451          a URL will abort the processing.  If set to 'ignore', errors
     452          are ignored.  If set to 'discard', failing URLs will be removed.
     453          """
     454          if base_url is None:
     455              base_url = self.base_url
     456              if base_url is None:
     457                  raise TypeError(
     458                      "No base_url given, and the document has no base_url")
     459          if resolve_base_href:
     460              self.resolve_base_href()
     461
     462          if handle_failures == 'ignore':
     463              def link_repl(href):
     464                  try:
     465                      return urljoin(base_url, href)
     466                  except ValueError:
     467                      return href
     468          elif handle_failures == 'discard':
     469              def link_repl(href):
     470                  try:
     471                      return urljoin(base_url, href)
     472                  except ValueError:
     473                      return None
     474          elif handle_failures is None:
     475              def link_repl(href):
     476                  return urljoin(base_url, href)
     477          else:
     478              raise ValueError(
     479                  "unexpected value for handle_failures: %r" % handle_failures)
     480
     481          self.rewrite_links(link_repl)
     482
     483 -    def resolve_base_href(self, handle_failures=None):
     484          """
     485          Find any ``<base href>`` tag in the document, and apply its
     486          values to all links found in the document.  Also remove the
     487          tag once it has been applied.
     488
     489          If ``handle_failures`` is None (default), a failure to process
     490          a URL will abort the processing.  If set to 'ignore', errors
     491          are ignored.  If set to 'discard', failing URLs will be removed.
     492          """
     493          base_href = None
     494          basetags = self.xpath('//base[@href]|//x:base[@href]',
     495                                namespaces={'x': XHTML_NAMESPACE})
     496          for b in basetags:
     497              base_href = b.get('href')
     498              b.drop_tree()
     499          if not base_href:
     500              return
     501          self.make_links_absolute(base_href, resolve_base_href=False,
     502                                   handle_failures=handle_failures)
     503
     504 -    def iterlinks(self):
     505          """
     506          Yield (element, attribute, link, pos), where attribute may be None
     507          (indicating the link is in the text).  ``pos`` is the position
     508          where the link occurs; often 0, but sometimes something else in
     509          the case of links in stylesheets or style tags.
     510
     511          Note: <base href> is *not* taken into account in any way.  The
     512          link you get is exactly the link in the document.
     513
     514          Note: multiple links inside of a single text string or
     515          attribute value are returned in reversed order.  This makes it
     516          possible to replace or delete them from the text string value
     517          based on their reported text positions.  Otherwise, a
     518          modification at one text position can change the positions of
     519          links reported later on.
     520          """
     521          link_attrs = defs.link_attrs
     522          for el in self.iter(etree.Element):
     523              attribs = el.attrib
     524              tag = _nons(el.tag)
     525              if tag == 'object':
     526                  codebase = None
     527                  ## <object> tags have attributes that are relative to
     528                  ## codebase
     529                  if 'codebase' in attribs:
     530                      codebase = el.get('codebase')
     531                      yield (el, 'codebase', codebase, 0)
     532                  for attrib in ('classid', 'data'):
     533                      if attrib in attribs:
     534                          value = el.get(attrib)
     535                          if codebase is not None:
     536                              value = urljoin(codebase, value)
     537                          yield (el, attrib, value, 0)
     538                  if 'archive' in attribs:
     539                      for match in _archive_re.finditer(el.get('archive')):
     540                          value = match.group(0)
     541                          if codebase is not None:
     542                              value = urljoin(codebase, value)
     543                          yield (el, 'archive', value, match.start())
     544              else:
     545                  for attrib in link_attrs:
     546                      if attrib in attribs:
     547                          yield (el, attrib, attribs[attrib], 0)
     548              if tag == 'meta':
     549                  http_equiv = attribs.get('http-equiv', '').lower()
     550                  if http_equiv == 'refresh':
     551                      content = attribs.get('content', '')
     552                      match = _parse_meta_refresh_url(content)
     553                      url = (match.group('url') if match else content).strip()
     554                      # unexpected content means the redirect won't work, but we might
     555                      # as well be permissive and return the entire string.
     556                      if url:
     557                          url, pos = _unquote_match(
     558                              url, match.start('url') if match else content.find(url))
     559                          yield (el, 'content', url, pos)
     560              elif tag == 'param':
     561                  valuetype = el.get('valuetype') or ''
     562                  if valuetype.lower() == 'ref':
     563                      ## FIXME: while it's fine we *find* this link,
     564                      ## according to the spec we aren't supposed to
     565                      ## actually change the value, including resolving
     566                      ## it.  It can also still be a link, even if it
     567                      ## doesn't have a valuetype="ref" (which seems to be the norm)
     568                      ## http://www.w3.org/TR/html401/struct/objects.html#adef-valuetype
     569                      yield (el, 'value', el.get('value'), 0)
     570              elif tag == 'style' and el.text:
     571                  urls = [
     572                      # (start_pos, url)
     573                      _unquote_match(match.group(1), match.start(1))[::-1]
     574                      for match in _iter_css_urls(el.text)
     575                      ] + [
     576                      (match.start(1), match.group(1))
     577                      for match in _iter_css_imports(el.text)
     578                      ]
     579                  if urls:
     580                      # sort by start pos to bring both match sets back into order
     581                      # and reverse the list to report correct positions despite
     582                      # modifications
     583                      urls.sort(reverse=True)
     584                      for start, url in urls:
     585                          yield (el, None, url, start)
     586              if 'style' in attribs:
     587                  urls = list(_iter_css_urls(attribs['style']))
     588                  if urls:
     589                      # return in reversed order to simplify in-place modifications
     590                      for match in urls[::-1]:
     591                          url, start = _unquote_match(match.group(1), match.start(1))
     592                          yield (el, 'style', url, start)
     593
     594 -    def rewrite_links(self, link_repl_func, resolve_base_href=True,
     595                        base_href=None):
     596          """
     597          Rewrite all the links in the document.  For each link
     598          ``link_repl_func(link)`` will be called, and the return value
     599          will replace the old link.
     600
     601          Note that links may not be absolute (unless you first called
     602          ``make_links_absolute()``), and may be internal (e.g.,
     603          ``'#anchor'``).  They can also be values like
     604          ``'mailto:email'`` or ``'javascript:expr'``.
     605
     606          If you give ``base_href`` then all links passed to
     607          ``link_repl_func()`` will take that into account.
     608
     609          If the ``link_repl_func`` returns None, the attribute or
     610          tag text will be removed completely.
     611          """
     612          if base_href is not None:
     613              # FIXME: this can be done in one pass with a wrapper
     614              # around link_repl_func
     615              self.make_links_absolute(
     616                  base_href, resolve_base_href=resolve_base_href)
     617          elif resolve_base_href:
     618              self.resolve_base_href()
     619
     620          for el, attrib, link, pos in self.iterlinks():
     621              new_link = link_repl_func(link.strip())
     622              if new_link == link:
     623                  continue
     624              if new_link is None:
     625                  # Remove the attribute or element content
     626                  if attrib is None:
     627                      el.text = ''
     628                  else:
     629                      del el.attrib[attrib]
     630                  continue
     631
     632              if attrib is None:
     633                  new = el.text[:pos] + new_link + el.text[pos+len(link):]
     634                  el.text = new
     635              else:
     636                  cur = el.get(attrib)
     637                  if not pos and len(cur) == len(link):
     638                      new = new_link  # most common case
     639                  else:
     640                      new = cur[:pos] + new_link + cur[pos+len(link):]
     641                  el.set(attrib, new)
     642
     643
     644 -class _MethodFunc(object):
     645      """
     646      An object that represents a method on an element as a function;
     647      the function takes either an element or an HTML string.  It
     648      returns whatever the function normally returns, or if the function
     649      works in-place (and so returns None) it returns a serialized form
     650      of the resulting document.
     651      """
     652 -    def __init__(self, name, copy=False, source_class=HtmlMixin):
     653          self.name = name
     654          self.copy = copy
     655          self.__doc__ = getattr(source_class, self.name).__doc__
     656 -    def __call__(self, doc, *args, **kw):
     657          result_type = type(doc)
     658          if isinstance(doc, basestring):
     659              if 'copy' in kw:
     660                  raise TypeError(
     661                      "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name)
     662              doc = fromstring(doc, **kw)
     663          else:
     664              if 'copy' in kw:
     665                  make_a_copy = kw.pop('copy')
     666              else:
     667                  make_a_copy = self.copy
     668              if make_a_copy:
     669                  doc = copy.deepcopy(doc)
     670          meth = getattr(doc, self.name)
     671          result = meth(*args, **kw)
     672          # FIXME: this None test is a bit sloppy
     673          if result is None:
     674              # Then return what we got in
     675              return _transform_result(result_type, doc)
     676          else:
     677              return result
     678
     679
     680  find_rel_links = _MethodFunc('find_rel_links', copy=False)
     681  find_class = _MethodFunc('find_class', copy=False)
     682  make_links_absolute = _MethodFunc('make_links_absolute', copy=True)
     683  resolve_base_href = _MethodFunc('resolve_base_href', copy=True)
     684  iterlinks = _MethodFunc('iterlinks', copy=False)
     685  rewrite_links = _MethodFunc('rewrite_links', copy=True)
     686
     687
     688 -class HtmlComment(etree.CommentBase, HtmlMixin):
     689      pass
     690
     691
     692 -class HtmlElement(etree.ElementBase, HtmlMixin):
     693      # Override etree.ElementBase.cssselect() and set(), despite the MRO (FIXME: change base order?)
     694      cssselect = HtmlMixin.cssselect
     695      set = HtmlMixin.set
     696
     697
     698 -class HtmlProcessingInstruction(etree.PIBase, HtmlMixin):
     699      pass
     700
     701
     702 -class HtmlEntity(etree.EntityBase, HtmlMixin):
     703      pass
     704
     705
     706 -class HtmlElementClassLookup(etree.CustomElementClassLookup):
     707      """A lookup scheme for HTML Element classes.
     708
     709      To create a lookup instance with different Element classes, pass a tag
     710      name mapping of Element classes in the ``classes`` keyword argument and/or
     711      a tag name mapping of Mixin classes in the ``mixins`` keyword argument.
     712      The special key '*' denotes a Mixin class that should be mixed into all
     713      Element classes.
     714      """
     715      _default_element_classes = {}
     716
     717 -    def __init__(self, classes=None, mixins=None):
     718          etree.CustomElementClassLookup.__init__(self)
     719          if classes is None:
     720              classes = self._default_element_classes.copy()
     721          if mixins:
     722              mixers = {}
     723              for name, value in mixins:
     724                  if name == '*':
     725                      for n in classes.keys():
     726                          mixers.setdefault(n, []).append(value)
     727                  else:
     728                      mixers.setdefault(name, []).append(value)
     729              for name, mix_bases in mixers.items():
     730                  cur = classes.get(name, HtmlElement)
     731                  bases = tuple(mix_bases + [cur])
     732                  classes[name] = type(cur.__name__, bases, {})
     733          self._element_classes = classes
     734
     735 -    def lookup(self, node_type, document, namespace, name):
     736          if node_type == 'element':
     737              return self._element_classes.get(name.lower(), HtmlElement)
     738          elif node_type == 'comment':
     739              return HtmlComment
     740          elif node_type == 'PI':
     741              return HtmlProcessingInstruction
     742          elif node_type == 'entity':
     743              return HtmlEntity
     744          # Otherwise normal lookup
     745          return None
     746
     747
     748  ################################################################################
     749  # parsing
     750  ################################################################################
     751
     752  _looks_like_full_html_unicode = re.compile(
     753      unicode(r'^\s*<(?:html|!doctype)'), re.I).match
     754  _looks_like_full_html_bytes = re.compile(
     755      r'^\s*<(?:html|!doctype)'.encode('ascii'), re.I).match
     756
     757
     758 -def document_fromstring(html, parser=None, ensure_head_body=False, **kw):
     759      if parser is None:
     760          parser = html_parser
     761      value = etree.fromstring(html, parser, **kw)
     762      if value is None:
     763          raise etree.ParserError(
     764              "Document is empty")
     765      if ensure_head_body and value.find('head') is None:
     766          value.insert(0, Element('head'))
     767      if ensure_head_body and value.find('body') is None:
     768          value.append(Element('body'))
     769      return value
     770
     771
     772 -def fragments_fromstring(html, no_leading_text=False, base_url=None,
     773                           parser=None, **kw):
     774      """Parses several HTML elements, returning a list of elements.
     775
     776      The first item in the list may be a string.
     777      If no_leading_text is true, then it will be an error if there is
     778      leading text, and it will always be a list of only elements.
     779
     780      base_url will set the document's base_url attribute
     781      (and the tree's docinfo.URL).
     782      """
     783      if parser is None:
     784          parser = html_parser
     785      # FIXME: check what happens when you give html with a body, head, etc.
     786      if isinstance(html, bytes):
     787          if not _looks_like_full_html_bytes(html):
     788              # can't use %-formatting in early Py3 versions
     789              html = ('<html><body>'.encode('ascii') + html +
     790                      '</body></html>'.encode('ascii'))
     791      else:
     792          if not _looks_like_full_html_unicode(html):
     793              html = '<html><body>%s</body></html>' % html
     794      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     795      assert _nons(doc.tag) == 'html'
     796      bodies = [e for e in doc if _nons(e.tag) == 'body']
     797      assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html))
     798      body = bodies[0]
     799      elements = []
     800      if no_leading_text and body.text and body.text.strip():
     801          raise etree.ParserError(
     802              "There is leading text: %r" % body.text)
     803      if body.text and body.text.strip():
     804          elements.append(body.text)
     805      elements.extend(body)
     806      # FIXME: removing the reference to the parent artificial document
     807      # would be nice
     808      return elements
     809
     810
     811 -def fragment_fromstring(html, create_parent=False, base_url=None,
     812                          parser=None, **kw):
     813      """
     814      Parses a single HTML element; it is an error if there is more than
     815      one element, or if anything but whitespace precedes or follows the
     816      element.
     817
     818      If ``create_parent`` is true (or is a tag name) then a parent node
     819      will be created to encapsulate the HTML in a single element.  In this
     820      case, leading or trailing text is also allowed, as are multiple elements
     821      as result of the parsing.
     822
     823      Passing a ``base_url`` will set the document's ``base_url`` attribute
     824      (and the tree's docinfo.URL).
     825      """
     826      if parser is None:
     827          parser = html_parser
     828
     829      accept_leading_text = bool(create_parent)
     830
     831      elements = fragments_fromstring(
     832          html, parser=parser, no_leading_text=not accept_leading_text,
     833          base_url=base_url, **kw)
     834
     835      if create_parent:
     836          if not isinstance(create_parent, basestring):
     837              create_parent = 'div'
     838          new_root = Element(create_parent)
     839          if elements:
     840              if isinstance(elements[0], basestring):
     841                  new_root.text = elements[0]
     842                  del elements[0]
     843              new_root.extend(elements)
     844          return new_root
     845
     846      if not elements:
     847          raise etree.ParserError('No elements found')
     848      if len(elements) > 1:
     849          raise etree.ParserError(
     850              "Multiple elements found (%s)"
     851              % ', '.join([_element_name(e) for e in elements]))
     852      el = elements[0]
     853      if el.tail and el.tail.strip():
     854          raise etree.ParserError(
     855              "Element followed by text: %r" % el.tail)
     856      el.tail = None
     857      return el
     858
     859
     860 -def fromstring(html, base_url=None, parser=None, **kw):
     861      """
     862      Parse the html, returning a single element/document.
     863
     864      This tries to minimally parse the chunk of text, without knowing if it
     865      is a fragment or a document.
     866
     867      base_url will set the document's base_url attribute (and the tree's docinfo.URL)
     868      """
     869      if parser is None:
     870          parser = html_parser
     871      if isinstance(html, bytes):
     872          is_full_html = _looks_like_full_html_bytes(html)
     873      else:
     874          is_full_html = _looks_like_full_html_unicode(html)
     875      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     876      if is_full_html:
     877          return doc
     878      # otherwise, lets parse it out...
     879      bodies = doc.findall('body')
     880      if not bodies:
     881          bodies = doc.findall('{%s}body' % XHTML_NAMESPACE)
     882      if bodies:
     883          body = bodies[0]
     884          if len(bodies) > 1:
     885              # Somehow there are multiple bodies, which is bad, but just
     886              # smash them into one body
     887              for other_body in bodies[1:]:
     888                  if other_body.text:
     889                      if len(body):
     890                          body[-1].tail = (body[-1].tail or '') + other_body.text
     891                      else:
     892                          body.text = (body.text or '') + other_body.text
     893                  body.extend(other_body)
     894                  # We'll ignore tail
     895                  # I guess we are ignoring attributes too
     896                  other_body.drop_tree()
     897      else:
     898          body = None
     899      heads = doc.findall('head')
     900      if not heads:
     901          heads = doc.findall('{%s}head' % XHTML_NAMESPACE)
     902      if heads:
     903          # Well, we have some sort of structure, so lets keep it all
     904          head = heads[0]
     905          if len(heads) > 1:
     906              for other_head in heads[1:]:
     907                  head.extend(other_head)
     908                  # We don't care about text or tail in a head
     909                  other_head.drop_tree()
     910          return doc
     911      if body is None:
     912          return doc
     913      if (len(body) == 1 and (not body.text or not body.text.strip())
     914          and (not body[-1].tail or not body[-1].tail.strip())):
     915          # The body has just one element, so it was probably a single
     916          # element passed in
     917          return body[0]
     918      # Now we have a body which represents a bunch of tags which have the
     919      # content that was passed in.  We will create a fake container, which
     920      # is the body tag, except <body> implies too much structure.
     921      if _contains_block_level_tag(body):
     922          body.tag = 'div'
     923      else:
     924          body.tag = 'span'
     925      return body
     926
     927
     928 -def parse(filename_or_url, parser=None, base_url=None, **kw):
     929      """
     930      Parse a filename, URL, or file-like object into an HTML document
     931      tree.  Note: this returns a tree, not an element.  Use
     932      ``parse(...).getroot()`` to get the document root.
     933
     934      You can override the base URL with the ``base_url`` keyword.  This
     935      is most useful when parsing from a file-like object.
     936      """
     937      if parser is None:
     938          parser = html_parser
     939      return etree.parse(filename_or_url, parser, base_url=base_url, **kw)
     940
     941
     942 -def _contains_block_level_tag(el):
     943      # FIXME: I could do this with XPath, but would that just be
     944      # unnecessarily slow?
     945      for el in el.iter(etree.Element):
     946          if _nons(el.tag) in defs.block_tags:
     947              return True
     948      return False
     949
     950
     951 -def _element_name(el):
     952      if isinstance(el, etree.CommentBase):
     953          return 'comment'
     954      elif isinstance(el, basestring):
     955          return 'string'
     956      else:
     957          return _nons(el.tag)
     958
     959
     960  ################################################################################
     961  # form handling
     962  ################################################################################
     963
     964 -class FormElement(HtmlElement):
     965      """
     966      Represents a <form> element.
     967      """
     968
     969      @property
     970 -    def inputs(self):
     971          """
     972          Returns an accessor for all the input elements in the form.
     973
     974          See `InputGetter` for more information about the object.
     975          """
     976          return InputGetter(self)
     977
     978      @property
     979 -    def fields(self):
     980          """
     981          Dictionary-like object that represents all the fields in this
     982          form.  You can set values in this dictionary to effect the
     983          form.
     984          """
     985          return FieldsDict(self.inputs)
     986
     987      @fields.setter
     988 -    def fields(self, value):
     989          fields = self.fields
     990          prev_keys = fields.keys()
     991          for key, value in value.items():
     992              if key in prev_keys:
     993                  prev_keys.remove(key)
     994              fields[key] = value
     995          for key in prev_keys:
     996              if key is None:
     997                  # Case of an unnamed input; these aren't really
     998                  # expressed in form_values() anyway.
     999                  continue
    1000              fields[key] = None
    1001
    1002 -    def _name(self):
    1003          if self.get('name'):
    1004              return self.get('name')
    1005          elif self.get('id'):
    1006              return '#' + self.get('id')
    1007          iter_tags = self.body.iter
    1008          forms = list(iter_tags('form'))
    1009          if not forms:
    1010              forms = list(iter_tags('{%s}form' % XHTML_NAMESPACE))
    1011          return str(forms.index(self))
    1012
    1013 -    def form_values(self):
    1014          """
    1015          Return a list of tuples of the field values for the form.
    1016          This is suitable to be passed to ``urllib.urlencode()``.
    1017          """
    1018          results = []
    1019          for el in self.inputs:
    1020              name = el.name
    1021              if not name or 'disabled' in el.attrib:
    1022                  continue
    1023              tag = _nons(el.tag)
    1024              if tag == 'textarea':
    1025                  results.append((name, el.value))
    1026              elif tag == 'select':
    1027                  value = el.value
    1028                  if el.multiple:
    1029                      for v in value:
    1030                          results.append((name, v))
    1031                  elif value is not None:
    1032                      results.append((name, el.value))
    1033              else:
    1034                  assert tag == 'input', (
    1035                      "Unexpected tag: %r" % el)
    1036                  if el.checkable and not el.checked:
    1037                      continue
    1038                  if el.type in ('submit', 'image', 'reset', 'file'):
    1039                      continue
    1040                  value = el.value
    1041                  if value is not None:
    1042                      results.append((name, el.value))
    1043          return results
    1044
    1045      @property
    1046 -    def action(self):
    1047          """
    1048          Get/set the form's ``action`` attribute.
    1049          """
    1050          base_url = self.base_url
    1051          action = self.get('action')
    1052          if base_url and action is not None:
    1053              return urljoin(base_url, action)
    1054          else:
    1055              return action
    1056
    1057      @action.setter
    1058 -    def action(self, value):
    1059          self.set('action', value)
    1060
    1061      @action.deleter
    1062 -    def action(self):
    1063          attrib = self.attrib
    1064          if 'action' in attrib:
    1065              del attrib['action']
    1066
    1067      @property
    1068 -    def method(self):
    1069          """
    1070          Get/set the form's method.  Always returns a capitalized
    1071          string, and defaults to ``'GET'``
    1072          """
    1073          return self.get('method', 'GET').upper()
    1074
    1075      @method.setter
    1076 -    def method(self, value):
    1077          self.set('method', value.upper())
    1078
    1079
    1080  HtmlElementClassLookup._default_element_classes['form'] = FormElement
    1081
    1082
    1083 -def submit_form(form, extra_values=None, open_http=None):
    1084      """
    1085      Helper function to submit a form.  Returns a file-like object, as from
    1086      ``urllib.urlopen()``.  This object also has a ``.geturl()`` function,
    1087      which shows the URL if there were any redirects.
    1088
    1089      You can use this like::
    1090
    1091          form = doc.forms[0]
    1092          form.inputs['foo'].value = 'bar' # etc
    1093          response = form.submit()
    1094          doc = parse(response)
    1095          doc.make_links_absolute(response.geturl())
    1096
    1097      To change the HTTP requester, pass a function as ``open_http`` keyword
    1098      argument that opens the URL for you.  The function must have the following
    1099      signature::
    1100
    1101          open_http(method, URL, values)
    1102
    1103      The action is one of 'GET' or 'POST', the URL is the target URL as a
    1104      string, and the values are a sequence of ``(name, value)`` tuples with the
    1105      form data.
    1106      """
    1107      values = form.form_values()
    1108      if extra_values:
    1109          if hasattr(extra_values, 'items'):
    1110              extra_values = extra_values.items()
    1111          values.extend(extra_values)
    1112      if open_http is None:
    1113          open_http = open_http_urllib
    1114      if form.action:
    1115          url = form.action
    1116      else:
    1117          url = form.base_url
    1118      return open_http(form.method, url, values)
    1119
    1120
    1121 -def open_http_urllib(method, url, values):
    1122      if not url:
    1123          raise ValueError("cannot submit, no URL provided")
    1124      ## FIXME: should test that it's not a relative URL or something
    1125      try:
    1126          from urllib import urlencode, urlopen
    1127      except ImportError: # Python 3
    1128          from urllib.request import urlopen
    1129          from urllib.parse import urlencode
    1130      if method == 'GET':
    1131          if '?' in url:
    1132              url += '&'
    1133          else:
    1134              url += '?'
    1135          url += urlencode(values)
    1136          data = None
    1137      else:
    1138          data = urlencode(values)
    1139          if not isinstance(data, bytes):
    1140              data = data.encode('ASCII')
    1141      return urlopen(url, data)
    1142
    1143
    1144 -class FieldsDict(MutableMapping):
    1145
    1146 -    def __init__(self, inputs):
    1147          self.inputs = inputs
    1148 -    def __getitem__(self, item):
    1149          return self.inputs[item].value
    1150 -    def __setitem__(self, item, value):
    1151          self.inputs[item].value = value
    1152 -    def __delitem__(self, item):
    1153          raise KeyError(
    1154              "You cannot remove keys from ElementDict")
    1155 -    def keys(self):
    1156          return self.inputs.keys()
    1157 -    def __contains__(self, item):
    1158          return item in self.inputs
    1159 -    def __iter__(self):
    1160          return iter(self.inputs.keys())
    1161 -    def __len__(self):
    1162          return len(self.inputs)
    1163
    1164 -    def __repr__(self):
    1165          return '<%s for form %s>' % (
    1166              self.__class__.__name__,
    1167              self.inputs.form._name())
    1168
    1169
    1170 -class InputGetter(object):
    1171
    1172      """
    1173      An accessor that represents all the input fields in a form.
    1174
    1175      You can get fields by name from this, with
    1176      ``form.inputs['field_name']``.  If there are a set of checkboxes
    1177      with the same name, they are returned as a list (a `CheckboxGroup`
    1178      which also allows value setting).  Radio inputs are handled
    1179      similarly.
    1180
    1181      You can also iterate over this to get all input elements.  This
    1182      won't return the same thing as if you get all the names, as
    1183      checkboxes and radio elements are returned individually.
    1184      """
    1185
    1186      _name_xpath = etree.XPath(".//*[@name = $name and (local-name(.) = 'select' or local-name(.) = 'input' or local-name(.) = 'textarea')]")
    1187      _all_xpath = etree.XPath(".//*[local-name() = 'select' or local-name() = 'input' or local-name() = 'textarea']")
    1188
    1189 -    def __init__(self, form):
    1190          self.form = form
    1191
    1192 -    def __repr__(self):
    1193          return '<%s for form %s>' % (
    1194              self.__class__.__name__,
    1195              self.form._name())
    1196
    1197      ## FIXME: there should be more methods, and it's unclear if this is
    1198      ## a dictionary-like object or list-like object
    1199
    1200 -    def __getitem__(self, name):
    1201          results = self._name_xpath(self.form, name=name)
    1202          if results:
    1203              type = results[0].get('type')
    1204              if type == 'radio' and len(results) > 1:
    1205                  group = RadioGroup(results)
    1206                  group.name = name
    1207                  return group
    1208              elif type == 'checkbox' and len(results) > 1:
    1209                  group = CheckboxGroup(results)
    1210                  group.name = name
    1211                  return group
    1212              else:
    1213                  # I don't like throwing away elements like this
    1214                  return results[0]
    1215          else:
    1216              raise KeyError(
    1217                  "No input element with the name %r" % name)
    1218
    1219 -    def __contains__(self, name):
    1220          results = self._name_xpath(self.form, name=name)
    1221          return bool(results)
    1222
    1223 -    def keys(self):
    1224          names = set()
    1225          for el in self:
    1226              names.add(el.name)
    1227          if None in names:
    1228              names.remove(None)
    1229          return list(names)
    1230
    1231 -    def __iter__(self):
    1232          ## FIXME: kind of dumb to turn a list into an iterator, only
    1233          ## to have it likely turned back into a list again :(
    1234          return iter(self._all_xpath(self.form))
    1235
    1236
    1237 -class InputMixin(object):
    1238      """
    1239      Mix-in for all input elements (input, select, and textarea)
    1240      """
    1241      @property
    1242 -    def name(self):
    1243          """
    1244          Get/set the name of the element
    1245          """
    1246          return self.get('name')
    1247
    1248      @name.setter
    1249 -    def name(self, value):
    1250          self.set('name', value)
    1251
    1252      @name.deleter
    1253 -    def name(self):
    1254          attrib = self.attrib
    1255          if 'name' in attrib:
    1256              del attrib['name']
    1257
    1258 -    def __repr__(self):
    1259          type_name = getattr(self, 'type', None)
    1260          if type_name:
    1261              type_name = ' type=%r' % type_name
    1262          else:
    1263              type_name = ''
    1264          return '<%s %x name=%r%s>' % (
    1265              self.__class__.__name__, id(self), self.name, type_name)
    1266
    1267
    1268 -class TextareaElement(InputMixin, HtmlElement):
    1269      """
    1270      ``<textarea>`` element.  You can get the name with ``.name`` and
    1271      get/set the value with ``.value``
    1272      """
    1273      @property
    1274 -    def value(self):
    1275          """
    1276          Get/set the value (which is the contents of this element)
    1277          """
    1278          content = self.text or ''
    1279          if self.tag.startswith("{%s}" % XHTML_NAMESPACE):
    1280              serialisation_method = 'xml'
    1281          else:
    1282              serialisation_method = 'html'
    1283          for el in self:
    1284              # it's rare that we actually get here, so let's not use ''.join()
    1285              content += etree.tostring(
    1286                  el, method=serialisation_method, encoding='unicode')
    1287          return content
    1288
    1289      @value.setter
    1290 -    def value(self, value):
    1291          del self[:]
    1292          self.text = value
    1293
    1294      @value.deleter
    1295 -    def value(self):
    1296          self.text = ''
    1297          del self[:]
    1298
    1299
    1300  HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement
    1301
    1302
    1303 -class SelectElement(InputMixin, HtmlElement):
    1304      """
    1305      ``<select>`` element.  You can get the name with ``.name``.
    1306
    1307      ``.value`` will be the value of the selected option, unless this
    1308      is a multi-select element (``<select multiple>``), in which case
    1309      it will be a set-like object.  In either case ``.value_options``
    1310      gives the possible values.
    1311
    1312      The boolean attribute ``.multiple`` shows if this is a
    1313      multi-select.
    1314      """
    1315      @property
    1316 -    def value(self):
    1317          """
    1318          Get/set the value of this select (the selected option).
    1319
    1320          If this is a multi-select, this is a set-like object that
    1321          represents all the selected options.
    1322          """
    1323          if self.multiple:
    1324              return MultipleSelectOptions(self)
    1325          options = _options_xpath(self)
    1326
    1327          try:
    1328              selected_option = next(el for el in reversed(options) if el.get('selected') is not None)
    1329          except StopIteration:
    1330              try:
    1331                  selected_option = next(el for el in options if el.get('disabled') is None)
    1332              except StopIteration:
    1333                  return None
    1334          value = selected_option.get('value')
    1335          if value is None:
    1336              value = (selected_option.text or '').strip()
    1337          return value
    1338
    1339      @value.setter
    1340 -    def value(self, value):
    1341          if self.multiple:
    1342              if isinstance(value, basestring):
    1343                  raise TypeError("You must pass in a sequence")
    1344              values = self.value
    1345              values.clear()
    1346              values.update(value)
    1347              return
    1348          checked_option = None
    1349          if value is not None:
    1350              for el in _options_xpath(self):
    1351                  opt_value = el.get('value')
    1352                  if opt_value is None:
    1353                      opt_value = (el.text or '').strip()
    1354                  if opt_value == value:
    1355                      checked_option = el
    1356                      break
    1357              else:
    1358                  raise ValueError(
    1359                      "There is no option with the value of %r" % value)
    1360          for el in _options_xpath(self):
    1361              if 'selected' in el.attrib:
    1362                  del el.attrib['selected']
    1363          if checked_option is not None:
    1364              checked_option.set('selected', '')
    1365
    1366      @value.deleter
    1367 -    def value(self):
    1368          # FIXME: should del be allowed at all?
    1369          if self.multiple:
    1370              self.value.clear()
    1371          else:
    1372              self.value = None
    1373
    1374      @property
    1375 -    def value_options(self):
    1376          """
    1377          All the possible values this select can have (the ``value``
    1378          attribute of all the ``<option>`` elements.
    1379          """
    1380          options = []
    1381          for el in _options_xpath(self):
    1382              value = el.get('value')
    1383              if value is None:
    1384                  value = (el.text or '').strip()
    1385              options.append(value)
    1386          return options
    1387
    1388      @property
    1389 -    def multiple(self):
    1390          """
    1391          Boolean attribute: is there a ``multiple`` attribute on this element.
    1392          """
    1393          return 'multiple' in self.attrib
    1394
    1395      @multiple.setter
    1396 -    def multiple(self, value):
    1397          if value:
    1398              self.set('multiple', '')
    1399          elif 'multiple' in self.attrib:
    1400              del self.attrib['multiple']
    1401
    1402
    1403  HtmlElementClassLookup._default_element_classes['select'] = SelectElement
    1404
    1405
    1406 -class MultipleSelectOptions(SetMixin):
    1407      """
    1408      Represents all the selected options in a ``<select multiple>`` element.
    1409
    1410      You can add to this set-like option to select an option, or remove
    1411      to unselect the option.
    1412      """
    1413
    1414 -    def __init__(self, select):
    1415          self.select = select
    1416
    1417      @property
    1418 -    def options(self):
    1419          """
    1420          Iterator of all the ``<option>`` elements.
    1421          """
    1422          return iter(_options_xpath(self.select))
    1423
    1424 -    def __iter__(self):
    1425          for option in self.options:
    1426              if 'selected' in option.attrib:
    1427                  opt_value = option.get('value')
    1428                  if opt_value is None:
    1429                      opt_value = (option.text or '').strip()
    1430                  yield opt_value
    1431
    1432 -    def add(self, item):
    1433          for option in self.options:
    1434              opt_value = option.get('value')
    1435              if opt_value is None:
    1436                  opt_value = (option.text or '').strip()
    1437              if opt_value == item:
    1438                  option.set('selected', '')
    1439                  break
    1440          else:
    1441              raise ValueError(
    1442                  "There is no option with the value %r" % item)
    1443
    1444 -    def remove(self, item):
    1445          for option in self.options:
    1446              opt_value = option.get('value')
    1447              if opt_value is None:
    1448                  opt_value = (option.text or '').strip()
    1449              if opt_value == item:
    1450                  if 'selected' in option.attrib:
    1451                      del option.attrib['selected']
    1452                  else:
    1453                      raise ValueError(
    1454                          "The option %r is not currently selected" % item)
    1455                  break
    1456          else:
    1457              raise ValueError(
    1458                  "There is not option with the value %r" % item)
    1459
    1460 -    def __repr__(self):
    1461          return '<%s {%s} for select name=%r>' % (
    1462              self.__class__.__name__,
    1463              ', '.join([repr(v) for v in self]),
    1464              self.select.name)
    1465
    1466
    1467 -class RadioGroup(list):
    1468      """
    1469      This object represents several ``<input type=radio>`` elements
    1470      that have the same name.
    1471
    1472      You can use this like a list, but also use the property
    1473      ``.value`` to check/uncheck inputs.  Also you can use
    1474      ``.value_options`` to get the possible values.
    1475      """
    1476      @property
    1477 -    def value(self):
    1478          """
    1479          Get/set the value, which checks the radio with that value (and
    1480          unchecks any other value).
    1481          """
    1482          for el in self:
    1483              if 'checked' in el.attrib:
    1484                  return el.get('value')
    1485          return None
    1486
    1487      @value.setter
    1488 -    def value(self, value):
    1489          checked_option = None
    1490          if value is not None:
    1491              for el in self:
    1492                  if el.get('value') == value:
    1493                      checked_option = el
    1494                      break
    1495              else:
    1496                  raise ValueError("There is no radio input with the value %r" % value)
    1497          for el in self:
    1498              if 'checked' in el.attrib:
    1499                  del el.attrib['checked']
    1500          if checked_option is not None:
    1501              checked_option.set('checked', '')
    1502
    1503      @value.deleter
    1504 -    def value(self):
    1505          self.value = None
    1506
    1507      @property
    1508 -    def value_options(self):
    1509          """
    1510          Returns a list of all the possible values.
    1511          """
    1512          return [el.get('value') for el in self]
    1513
    1514 -    def __repr__(self):
    1515          return '%s(%s)' % (
    1516              self.__class__.__name__,
    1517              list.__repr__(self))
    1518
    1519
    1520 -class CheckboxGroup(list):
    1521      """
    1522      Represents a group of checkboxes (``<input type=checkbox>``) that
    1523      have the same name.
    1524
    1525      In addition to using this like a list, the ``.value`` attribute
    1526      returns a set-like object that you can add to or remove from to
    1527      check and uncheck checkboxes.  You can also use ``.value_options``
    1528      to get the possible values.
    1529      """
    1530      @property
    1531 -    def value(self):
    1532          """
    1533          Return a set-like object that can be modified to check or
    1534          uncheck individual checkboxes according to their value.
    1535          """
    1536          return CheckboxValues(self)
    1537
    1538      @value.setter
    1539 -    def value(self, value):
    1540          values = self.value
    1541          values.clear()
    1542          if not hasattr(value, '__iter__'):
    1543              raise ValueError(
    1544                  "A CheckboxGroup (name=%r) must be set to a sequence (not %r)"
    1545                  % (self[0].name, value))
    1546          values.update(value)
    1547
    1548      @value.deleter
    1549 -    def value(self):
    1550          self.value.clear()
    1551
    1552      @property
    1553 -    def value_options(self):
    1554          """
    1555          Returns a list of all the possible values.
    1556          """
    1557          return [el.get('value') for el in self]
    1558
    1559 -    def __repr__(self):
    1560          return '%s(%s)' % (
    1561              self.__class__.__name__, list.__repr__(self))
    1562
    1563
    1564 -class CheckboxValues(SetMixin):
    1565      """
    1566      Represents the values of the checked checkboxes in a group of
    1567      checkboxes with the same name.
    1568      """
    1569
    1570 -    def __init__(self, group):
    1571          self.group = group
    1572
    1573 -    def __iter__(self):
    1574          return iter([
    1575              el.get('value')
    1576              for el in self.group
    1577              if 'checked' in el.attrib])
    1578
    1579 -    def add(self, value):
    1580          for el in self.group:
    1581              if el.get('value') == value:
    1582                  el.set('checked', '')
    1583                  break
    1584          else:
    1585              raise KeyError("No checkbox with value %r" % value)
    1586
    1587 -    def remove(self, value):
    1588          for el in self.group:
    1589              if el.get('value') == value:
    1590                  if 'checked' in el.attrib:
    1591                      del el.attrib['checked']
    1592                  else:
    1593                      raise KeyError(
    1594                          "The checkbox with value %r was already unchecked" % value)
    1595                  break
    1596          else:
    1597              raise KeyError(
    1598                  "No checkbox with value %r" % value)
    1599
    1600 -    def __repr__(self):
    1601          return '<%s {%s} for checkboxes name=%r>' % (
    1602              self.__class__.__name__,
    1603              ', '.join([repr(v) for v in self]),
    1604              self.group.name)
    1605
    1606
    1607 -class InputElement(InputMixin, HtmlElement):
    1608      """
    1609      Represents an ``<input>`` element.
    1610
    1611      You can get the type with ``.type`` (which is lower-cased and
    1612      defaults to ``'text'``).
    1613
    1614      Also you can get and set the value with ``.value``
    1615
    1616      Checkboxes and radios have the attribute ``input.checkable ==
    1617      True`` (for all others it is false) and a boolean attribute
    1618      ``.checked``.
    1619
    1620      """
    1621
    1622      ## FIXME: I'm a little uncomfortable with the use of .checked
    1623      @property
    1624 -    def value(self):
    1625          """
    1626          Get/set the value of this element, using the ``value`` attribute.
    1627
    1628          Also, if this is a checkbox and it has no value, this defaults
    1629          to ``'on'``.  If it is a checkbox or radio that is not
    1630          checked, this returns None.
    1631          """
    1632          if self.checkable:
    1633              if self.checked:
    1634                  return self.get('value') or 'on'
    1635              else:
    1636                  return None
    1637          return self.get('value')
    1638
    1639      @value.setter
    1640 -    def value(self, value):
    1641          if self.checkable:
    1642              if not value:
    1643                  self.checked = False
    1644              else:
    1645                  self.checked = True
    1646                  if isinstance(value, basestring):
    1647                      self.set('value', value)
    1648          else:
    1649              self.set('value', value)
    1650
    1651      @value.deleter
    1652 -    def value(self):
    1653          if self.checkable:
    1654              self.checked = False
    1655          else:
    1656              if 'value' in self.attrib:
    1657                  del self.attrib['value']
    1658
    1659      @property
    1660 -    def type(self):
    1661          """
    1662          Return the type of this element (using the type attribute).
    1663          """
    1664          return self.get('type', 'text').lower()
    1665
    1666      @type.setter
    1667 -    def type(self, value):
    1668          self.set('type', value)
    1669
    1670      @property
    1671 -    def checkable(self):
    1672          """
    1673          Boolean: can this element be checked?
    1674          """
    1675          return self.type in ('checkbox', 'radio')
    1676
    1677      @property
    1678 -    def checked(self):
    1679          """
    1680          Boolean attribute to get/set the presence of the ``checked``
    1681          attribute.
    1682
    1683          You can only use this on checkable input types.
    1684          """
    1685          if not self.checkable:
    1686              raise AttributeError('Not a checkable input type')
    1687          return 'checked' in self.attrib
    1688
    1689      @checked.setter
    1690 -    def checked(self, value):
    1691          if not self.checkable:
    1692              raise AttributeError('Not a checkable input type')
    1693          if value:
    1694              self.set('checked', '')
    1695          else:
    1696              attrib = self.attrib
    1697              if 'checked' in attrib:
    1698                  del attrib['checked']
    1699
    1700
    1701  HtmlElementClassLookup._default_element_classes['input'] = InputElement
    1702
    1703
    1704 -class LabelElement(HtmlElement):
    1705      """
    1706      Represents a ``<label>`` element.
    1707
    1708      Label elements are linked to other elements with their ``for``
    1709      attribute.  You can access this element with ``label.for_element``.
    1710      """
    1711      @property
    1712 -    def for_element(self):
    1713          """
    1714          Get/set the element this label points to.  Return None if it
    1715          can't be found.
    1716          """
    1717          id = self.get('for')
    1718          if not id:
    1719              return None
    1720          return self.body.get_element_by_id(id)
    1721
    1722      @for_element.setter
    1723 -    def for_element(self, other):
    1724          id = other.get('id')
    1725          if not id:
    1726              raise TypeError(
    1727                  "Element %r has no id attribute" % other)
    1728          self.set('for', id)
    1729
    1730      @for_element.deleter
    1731 -    def for_element(self):
    1732          attrib = self.attrib
    1733          if 'id' in attrib:
    1734              del attrib['id']
    1735
    1736
    1737  HtmlElementClassLookup._default_element_classes['label'] = LabelElement
    1738
    1739
    1740  ############################################################
    1741  ## Serialization
    1742  ############################################################
    1743
    1744 -def html_to_xhtml(html):
    1745      """Convert all tags in an HTML tree to XHTML by moving them to the
    1746      XHTML namespace.
    1747      """
    1748      try:
    1749          html = html.getroot()
    1750      except AttributeError:
    1751          pass
    1752      prefix = "{%s}" % XHTML_NAMESPACE
    1753      for el in html.iter(etree.Element):
    1754          tag = el.tag
    1755          if tag[0] != '{':
    1756              el.tag = prefix + tag
    1757
    1758
    1759 -def xhtml_to_html(xhtml):
    1760      """Convert all tags in an XHTML tree to HTML by removing their
    1761      XHTML namespace.
    1762      """
    1763      try:
    1764          xhtml = xhtml.getroot()
    1765      except AttributeError:
    1766          pass
    1767      prefix = "{%s}" % XHTML_NAMESPACE
    1768      prefix_len = len(prefix)
    1769      for el in xhtml.iter(prefix + "*"):
    1770          el.tag = el.tag[prefix_len:]
    1771
    1772
    1773  # This isn't a general match, but it's a match for what libxml2
    1774  # specifically serialises:
    1775  __str_replace_meta_content_type = re.compile(
    1776      r'<meta http-equiv="Content-Type"[^>]*>').sub
    1777  __bytes_replace_meta_content_type = re.compile(
    1778      r'<meta http-equiv="Content-Type"[^>]*>'.encode('ASCII')).sub
    1779
    1780
    1781 -def tostring(doc, pretty_print=False, include_meta_content_type=False,
    1782               encoding=None, method="html", with_tail=True, doctype=None):
    1783      """Return an HTML string representation of the document.
    1784
    1785      Note: if include_meta_content_type is true this will create a
    1786      ``<meta http-equiv="Content-Type" ...>`` tag in the head;
    1787      regardless of the value of include_meta_content_type any existing
    1788      ``<meta http-equiv="Content-Type" ...>`` tag will be removed
    1789
    1790      The ``encoding`` argument controls the output encoding (defaults to
    1791      ASCII, with &#...; character references for any characters outside
    1792      of ASCII).  Note that you can pass the name ``'unicode'`` as
    1793      ``encoding`` argument to serialise to a Unicode string.
    1794
    1795      The ``method`` argument defines the output method.  It defaults to
    1796      'html', but can also be 'xml' for xhtml output, or 'text' to
    1797      serialise to plain text without markup.
    1798
    1799      To leave out the tail text of the top-level element that is being
    1800      serialised, pass ``with_tail=False``.
    1801
    1802      The ``doctype`` option allows passing in a plain string that will
    1803      be serialised before the XML tree.  Note that passing in non
    1804      well-formed content here will make the XML output non well-formed.
    1805      Also, an existing doctype in the document tree will not be removed
    1806      when serialising an ElementTree instance.
    1807
    1808      Example::
    1809
    1810          >>> from lxml import html
    1811          >>> root = html.fragment_fromstring('<p>Hello<br>world!</p>')
    1812
    1813          >>> html.tostring(root)
    1814          b'<p>Hello<br>world!</p>'
    1815          >>> html.tostring(root, method='html')
    1816          b'<p>Hello<br>world!</p>'
    1817
    1818          >>> html.tostring(root, method='xml')
    1819          b'<p>Hello<br/>world!</p>'
    1820
    1821          >>> html.tostring(root, method='text')
    1822          b'Helloworld!'
    1823
    1824          >>> html.tostring(root, method='text', encoding='unicode')
    1825          u'Helloworld!'
    1826
    1827          >>> root = html.fragment_fromstring('<div><p>Hello<br>world!</p>TAIL</div>')
    1828          >>> html.tostring(root[0], method='text', encoding='unicode')
    1829          u'Helloworld!TAIL'
    1830
    1831          >>> html.tostring(root[0], method='text', encoding='unicode', with_tail=False)
    1832          u'Helloworld!'
    1833
    1834          >>> doc = html.document_fromstring('<p>Hello<br>world!</p>')
    1835          >>> html.tostring(doc, method='html', encoding='unicode')
    1836          u'<html><body><p>Hello<br>world!</p></body></html>'
    1837
    1838          >>> print(html.tostring(doc, method='html', encoding='unicode',
    1839          ...          doctype='<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"'
    1840          ...                  ' "http://www.w3.org/TR/html4/strict.dtd">'))
    1841          <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
    1842          <html><body><p>Hello<br>world!</p></body></html>
    1843      """
    1844      html = etree.tostring(doc, method=method, pretty_print=pretty_print,
    1845                            encoding=encoding, with_tail=with_tail,
    1846                            doctype=doctype)
    1847      if method == 'html' and not include_meta_content_type:
    1848          if isinstance(html, str):
    1849              html = __str_replace_meta_content_type('', html)
    1850          else:
    1851              html = __bytes_replace_meta_content_type(bytes(), html)
    1852      return html
    1853
    1854
    1855  tostring.__doc__ = __fix_docstring(tostring.__doc__)
    1856
    1857
    1858 -def open_in_browser(doc, encoding=None):
    1859      """
    1860      Open the HTML document in a web browser, saving it to a temporary
    1861      file to open it.  Note that this does not delete the file after
    1862      use.  This is mainly meant for debugging.
    1863      """
    1864      import os
    1865      import webbrowser
    1866      import tempfile
    1867      if not isinstance(doc, etree._ElementTree):
    1868          doc = etree.ElementTree(doc)
    1869      handle, fn = tempfile.mkstemp(suffix='.html')
    1870      f = os.fdopen(handle, 'wb')
    1871      try:
    1872          doc.write(f, method="html", encoding=encoding or doc.docinfo.encoding or "UTF-8")
    1873      finally:
    1874          # we leak the file itself here, but we should at least close it
    1875          f.close()
    1876      url = 'file://' + fn.replace(os.path.sep, '/')
    1877      print(url)
    1878      webbrowser.open(url)
    1879
    1880
    1881  ################################################################################
    1882  # configure Element class lookup
    1883  ################################################################################
    1884
    1885 -class HTMLParser(etree.HTMLParser):
    1886      """An HTML parser that is configured to return lxml.html Element
    1887      objects.
    1888      """
    1889 -    def __init__(self, **kwargs):
    1890          super(HTMLParser, self).__init__(**kwargs)
    1891          self.set_element_class_lookup(HtmlElementClassLookup())
    1892
    1893
    1894 -class XHTMLParser(etree.XMLParser):
    1895      """An XML parser that is configured to return lxml.html Element
    1896      objects.
    1897
    1898      Note that this parser is not really XHTML aware unless you let it
    1899      load a DTD that declares the HTML entities.  To do this, make sure
    1900      you have the XHTML DTDs installed in your catalogs, and create the
    1901      parser like this::
    1902
    1903          >>> parser = XHTMLParser(load_dtd=True)
    1904
    1905      If you additionally want to validate the document, use this::
    1906
    1907          >>> parser = XHTMLParser(dtd_validation=True)
    1908
    1909      For catalog support, see http://www.xmlsoft.org/catalog.html.
    1910      """
    1911 -    def __init__(self, **kwargs):
    1912          super(XHTMLParser, self).__init__(**kwargs)
    1913          self.set_element_class_lookup(HtmlElementClassLookup())
    1914
    1915
    1916 -def Element(*args, **kw):
    1917      """Create a new HTML Element.
    1918
    1919      This can also be used for XHTML documents.
    1920      """
    1921      v = html_parser.makeelement(*args, **kw)
    1922      return v
    1923
    1924
    1925  html_parser = HTMLParser()
    1926  xhtml_parser = XHTMLParser()
    1927