staticoso/Source/Libs/htmlmin/parser.py

"""
Copyright (c) 2013, Dave Mankoff
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of Dave Mankoff nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL DAVE MANKOFF BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

from __future__ import unicode_literals
import logging
import sys

import re
from .python3html.parser import HTMLParser

from . import escape

# https://www.w3.org/TR/html5/single-page.html#space-character
HTML_SPACE_RE = re.compile('[\x20\x09\x0a\x0c\x0d]+')
HTML_ALL_SPACE_RE = re.compile('^[\x20\x09\x0a\x0c\x0d]+$')
HTML_LEADING_SPACE_RE = re.compile(
  '^[\x20\x09\x0a\x0c\x0d]+')
HTML_TRAILING_SPACE_RE = re.compile(
  '[\x20\x09\x0a\x0c\x0d]+$')
HTML_LEADING_TRAILING_SPACE_RE = re.compile(
  '(^[\x20\x09\x0a\x0c\x0d]+)|([\x20\x09\x0a\x0c\x0d]+$)')

PRE_TAGS = ('pre', 'textarea')  # styles and scripts are never minified
# http://www.w3.org/TR/html51/syntax.html#elements-0
NO_CLOSE_TAGS = ('area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img',
                 'input', 'keygen', 'link', 'meta', 'param', 'source', 'track',
                 'wbr')
# http://www.w3.org/TR/html51/index.html#attributes-1
BOOLEAN_ATTRIBUTES = {
  'audio': ('autoplay', 'controls', 'hidden', 'loop', 'muted',),
  'button': ('autofocus', 'disabled', 'formnovalidate', 'hidden',),
  'command': ('checked', 'disabled', 'hidden'),
  'dialog': ('hidden', 'open',),
  'fieldset': ('disabled', 'hidden',),
  'form': ('hidden', 'novalidate',),
  'iframe': ('hidden', 'seamless',),
  'img': ('hidden', 'ismap',),
  'input': ('autofocus', 'checked', 'disabled', 'formnovalidate', 'hidden',
            'multiple', 'readonly', 'required',),
  'keygen': ('autofocus', 'disabled', 'hidden',),
  'object': ('hidden', 'typesmustmatch',),
  'ol': ('hidden', 'reversed',),
  'optgroup': ('disabled', 'hidden',),
  'option': ('disabled', 'hidden', 'selected',),
  'script': ('async', 'defer', 'hidden',),
  'select': ('autofocus', 'disabled', 'hidden', 'multiple', 'required',),
  'style': ('hidden', 'scoped',),
  'textarea': ('autofocus', 'disabled', 'hidden', 'readonly', 'required',),
  'track': ('default', 'hidden', ),
  'video': ('autoplay', 'controls', 'hidden', 'loop', 'muted',),
  '*': ('hidden',),
}

# a list of tags and tags that they are closed by
TAG_SETS = {
  'li': ('li',),
  'dd': ('dd', 'dt'),
  'rp': ('rp', 'rt'),
  'p': ('address', 'article', 'aside', 'blockquote', 'dir', 'div', 'dl',
        'fieldset', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
        'header', 'hgroup', 'hr', 'menu', 'nav', 'ol', 'p', 'pre', 'section',
        'table', 'ul'),
  'optgroup': ('optgroup',),
  'option': ('option', 'optgroup'),
  'colgroup': '*',
  'tbody': ('tbody', 'tfoot'),
  'tfoot': ('tbody',),
  'tr': ('tr',),
  'td': ('td', 'th'),
}
TAG_SETS['dt'] = TAG_SETS['dd']
TAG_SETS['rt'] = TAG_SETS['rp']
TAG_SETS['thead'] = TAG_SETS['tbody']
TAG_SETS['th'] = TAG_SETS['td']

# Tag omission rules:
# http://www.w3.org/TR/html51/syntax.html#optional-tags

class HTMLMinError(Exception): pass
class ParseError(HTMLMinError): pass
class OpenTagNotFoundError(ParseError): pass

class HTMLMinParser(HTMLParser):
  def __init__(self,
               remove_comments=False,
               remove_empty_space=False,
               remove_all_empty_space=False,
               reduce_empty_attributes=True,
               reduce_boolean_attributes=False,
               remove_optional_attribute_quotes=True,
               convert_charrefs=True,
               keep_pre=False,
               pre_tags=PRE_TAGS,
               pre_attr='pre'):
    if sys.version_info[0] >= 3 and sys.version_info[1] >= 4:
      # convert_charrefs is True by default in Python 3.5.0 and newer. It was
      # introduced in 3.4.
      HTMLParser.__init__(self, convert_charrefs=False)
    else:
      HTMLParser.__init__(self)
    self.keep_pre = keep_pre
    self.pre_tags = pre_tags
    self.remove_comments = remove_comments
    self.remove_empty_space = remove_empty_space
    self.remove_all_empty_space = remove_all_empty_space
    self.reduce_empty_attributes = reduce_empty_attributes
    self.reduce_boolean_attributes = reduce_boolean_attributes
    self.remove_optional_attribute_quotes = remove_optional_attribute_quotes
    self.convert_charrefs = convert_charrefs
    self.pre_attr = pre_attr
    self.reset()

  def _tag_lang(self):
    return self._tag_stack[0][2] if self._tag_stack else None

  def build_tag(self, tag, attrs, close_tag):
    has_pre = False

    if self.reduce_boolean_attributes:
      bool_attrs = BOOLEAN_ATTRIBUTES.get(tag, BOOLEAN_ATTRIBUTES['*'])
    else:
      bool_attrs = False

    lang = self._tag_lang()
    attrs = list(attrs)  # We're modifying it in place
    last_quoted = last_no_slash = i = -1
    for k, v in attrs:
      pre_prefix = k.startswith("{}-".format(self.pre_attr))
      if pre_prefix:
        k = k[len(self.pre_attr)+1:]
      if k == self.pre_attr:
        has_pre = True
        if not self.keep_pre and not pre_prefix:
          continue
      if v and self.convert_charrefs and not pre_prefix:
        v = HTMLParser.unescape(self, v)
      if k == 'lang':
        lang = v
        if v == self._tag_lang():
          continue

      i += 1
      if not pre_prefix:
        k = escape.escape_attr_name(k)
      if (v is None or (not v and self.reduce_empty_attributes) or
          (bool_attrs and k in bool_attrs)):
        # For our use case, we treat boolean attributes as quoted because they
        # don't require space between them and "/>" in closing tags.
        attrs[i] = k
        last_quoted = i
      else:
        if pre_prefix:
          has_double_quotes = '"' in v
          has_single_quotes = "'" in v
          if not has_double_quotes:
            if not has_single_quotes and self.remove_optional_attribute_quotes:
              q = escape.NO_QUOTES
            else:
              q = escape.DOUBLE_QUOTE
          elif not has_single_quotes:
            q = escape.SINGLE_QUOTES
          else:
            logging.error('Unsafe content found in pre-attribute. Escaping.')
            (v, q) = escape.escape_attr_value(
              v, double_quote=not self.remove_optional_attribute_quotes)
        else:
          (v, q) = escape.escape_attr_value(
            v, double_quote=not self.remove_optional_attribute_quotes)
        if q == escape.NO_QUOTES:
          attrs[i] = '%s=%s' % (k, v)
          if v[-1] != '/':
            last_no_slash = i
        else:
          q = '"' if q == escape.DOUBLE_QUOTE else "'"
          attrs[i] = '%s=%s%s%s' % (k, q, v, q)
          last_quoted = i

    i += 1
    if i != len(attrs):
      del attrs[i:]

    # 1. If there are no attributes, no additional space is necessary.
    # 2. If last attribute is quoted, no additional space is necessary.
    # 3. Two things are happening here:
    #    a) according to the standard, <foo bar=baz/> should be treated as <foo
    #       bar="baz/"> so space is necessary if this is self-closing tag,
    #       however
    #    b) reportedly (https://github.com/mankyd/htmlmin/pull/12), older
    #       versions of WebKit interpret <foo bar=baz/> as self-closing tag so
    #       we need the space if the last argument ends with a slash.
    space_maybe = ''
    if attrs:
      needs_space = lambda last_attr: (last_attr[-1] not in '"\'' and
                                       (close_tag or last_attr[-1] == '/'))
      if needs_space(attrs[-1][-1]):
        # If moving attributes around can help, do it.  Otherwise bite the
        # bullet and put the space in.
        i = last_no_slash if last_quoted == -1 else last_quoted
        if i == -1 or needs_space(attrs[i]):
          space_maybe = ' '
        else:
          attrs.append(attrs[i])
          del attrs[i]

    return has_pre, '<%s%s%s%s%s>' % (escape.escape_tag(tag),
                                      ' ' if attrs else '',
                                      ' '.join(attrs),
                                      space_maybe,
                                      '/' if close_tag else ''), lang

  def handle_decl(self, decl):
    if (len(self._data_buffer) == 1 and 
        HTML_SPACE_RE.match(self._data_buffer[0][0])):
      self._data_buffer = []
    self._data_buffer.append('<!' + decl + '>')
    self._after_doctype = True

  def _close_tags_up_to(self, tag):
    num_pres = 0
    i = 0
    for i, t in enumerate(self._tag_stack):
      if t[1]:
        num_pres += 1
      if t[0] == tag:
        break

      # Only the html tag can close out everything. Put on the brakes if
      # we encounter a closing tag that we didn't recognize.
      if tag != 'html' and t[0] in ('body', 'html', 'head'):
        raise OpenTagNotFoundError()

    self._tag_stack = self._tag_stack[i+1:]

    return num_pres

  def handle_starttag(self, tag, attrs):
    self._after_doctype = False
    if tag == 'head':
      self._in_head = True
    elif self._in_head and tag == 'title':
      self._in_title = True
      self._title_newly_opened = True

    for t in self._tag_stack:
      closed_by_tags = TAG_SETS.get(t[0])
      if closed_by_tags and (closed_by_tags == '*' or tag in closed_by_tags):
        self._in_pre_tag -= self._close_tags_up_to(t[0])
        break

    has_pre, data, lang = self.build_tag(tag, attrs, False)
    start_pre = False
    if (has_pre or self._in_pre_tag > 0 or
        tag == 'script' or tag == 'style' or tag in self.pre_tags):
      self._in_pre_tag += 1
      start_pre = True

    self._tag_stack.insert(0, (tag, start_pre, lang))
    self._data_buffer.append(data)

  def handle_endtag(self, tag):
    # According to the spec, <p> tags don't get closed when a parent a
    # tag closes them. Here's some logic that addresses this.
    if tag == 'a':
      contains_p = False
      for i, t in enumerate(self._tag_stack):
        if t[0] == 'p':
          contains_p = True
        elif t[0] == 'a':
          break
      if contains_p: # the p tag, and all its children should be left open
        a_tag = self._tag_stack.pop(i)
        if a_tag[1]:
          self._in_pre_tag -= 1
    else:
      if tag == 'head':
        # TODO: Did we know that we were in an head tag?! If not, we need to
        # reminify everything to remove extra spaces.
        self._in_head = False
      elif tag == 'title':
        self._in_title = False
        self._title_newly_opened = False
      try:
        self._in_pre_tag -= self._close_tags_up_to(tag)
      except OpenTagNotFoundError:
        # Some tags don't require a start tag. Most do. Either way, we leave
        # closing tags along since they affect output. For instance, a '</p>'
        # results in a '<p></p>' in Chrome.
        pass
    if tag not in NO_CLOSE_TAGS:
      self._data_buffer.extend(['</', escape.escape_tag(tag), '>'])

  def handle_startendtag(self, tag, attrs):
    self._after_doctype = False
    data = self.build_tag(tag, attrs, tag not in NO_CLOSE_TAGS)[1]
    self._data_buffer.append(data)

  def handle_comment(self, data):
    if not self.remove_comments or re.match(r'^(?:!|\[if\s)', data):
      self._data_buffer.append('<!--{}-->'.format(
          data[1:] if len(data) and data[0] == '!' else data))

  def handle_data(self, data):
    if self._in_pre_tag > 0:
      self._data_buffer.append(data)
    else:
      # remove_all_empty_space matches everything. remove_empty_space only
      # matches if there's a newline involved.
      if self.remove_all_empty_space or self._in_head or self._after_doctype:
        if HTML_ALL_SPACE_RE.match(data):
          return
      elif (self.remove_empty_space and HTML_ALL_SPACE_RE.match(data) and
            ('\n' in data or '\r' in data)):
        return

      # if we're in the title, remove leading and trailing whitespace.
      # note that the title may be parsed in chunks if entityref's or charrefs
      # are encountered.
      if self._in_title:
        if self.__title_trailing_whitespace:
          self._data_buffer.append(' ')
        self.__title_trailing_whitespace = (
          HTML_ALL_SPACE_RE.match(data[-1]) is not None)
        if self._title_newly_opened:
          self._title_newly_opened = False
          data = HTML_LEADING_TRAILING_SPACE_RE.sub('', data)
        else:
          data = HTML_TRAILING_SPACE_RE.sub(
            '', HTML_LEADING_TRAILING_SPACE_RE.sub(' ', data))

      data = HTML_SPACE_RE.sub(' ', data)
      if not data:
        return

      if self._in_pre_tag == 0 and self._data_buffer:
        # If we're not in a pre block, its possible that we append two spaces
        # together, which we want to avoid. For instance, if we remove a comment
        # from between two blocks of text: a <!-- B --> c => a  c.
        if data[0] == ' ' and self._data_buffer[-1][-1] == ' ':
          data = data[1:]
          if not data:
            return
      self._data_buffer.append(data)

  def handle_entityref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&{};'.format(data))

  def handle_charref(self, data):
    if self._in_title:
      if not self._title_newly_opened and self.__title_trailing_whitespace:
        self._data_buffer.append(' ')
        self.__title_trailing_whitespace = False
      self._title_newly_opened = False
    self._data_buffer.append('&#{};'.format(data))

  def handle_pi(self, data):
    self._data_buffer.append('<?' + data + '>')

  def unknown_decl(self, data):
    self._data_buffer.append('<![' + data + ']>')

  def reset(self):
    self._data_buffer = []
    self._in_pre_tag = 0
    self._in_head = False
    self._in_title = False
    self._after_doctype = False
    self._tag_stack = []
    self._title_newly_opened = False
    self.__title_trailing_whitespace = False
    HTMLParser.reset(self)

  def unescape(self, val):
    """Override this method so that we can handle char ref conversion ourself.
    """
    return val

  @property
  def result(self):
    return ''.join(self._data_buffer)