Merge pull request #412 from ihabunek/rich

Rich text simplification

.flake8

@@ -1,5 +1,4 @@
 [flake8]
 exclude=build,tests,tmp,venv,toot/tui/scroll.py
 ignore=E128,W503
-per-file-ignores=toot/tui/stubs/urwidgets.py:F401
 max-line-length=120

setup.py

@@ -31,7 +31,7 @@ setup(
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
         'Programming Language :: Python :: 3',
     ],
-    packages=['toot', 'toot.tui', 'toot.utils'],
+    packages=['toot', 'toot.tui', 'toot.tui.richtext', 'toot.utils'],
     python_requires=">=3.7",
     install_requires=[
         "requests>=2.13,<3.0",

tests/tui/test_rich_text.py

@@ -0,0 +1,45 @@
+from urwid import Divider, Filler, Pile
+from toot.tui.richtext import url_to_widget
+from urwidgets import Hyperlink, TextEmbed
+
+from toot.tui.richtext.richtext import html_to_widgets
+
+
+def test_url_to_widget():
+    url = "http://foo.bar"
+    embed_widget = url_to_widget(url)
+    assert isinstance(embed_widget, TextEmbed)
+
+    [(filler, length)] = embed_widget.embedded
+    assert length == len(url)
+    assert isinstance(filler, Filler)
+
+    link_widget: Hyperlink = filler.base_widget
+    assert isinstance(link_widget, Hyperlink)
+
+    assert link_widget.attrib == "link"
+    assert link_widget.text == url
+    assert link_widget.uri == url
+
+
+def test_html_to_widgets():
+    html = """
+    <p>foo</p>
+    <p>foo <b>bar</b> <i>baz</i></p>
+    """.strip()
+
+    [foo, divider, bar] = html_to_widgets(html)
+
+    assert isinstance(foo, Pile)
+    assert isinstance(divider, Divider)
+    assert isinstance(bar, Pile)
+
+    [foo_embed] = foo.widget_list
+    assert foo_embed.embedded == []
+    assert foo_embed.attrib == []
+    assert foo_embed.text == "foo"
+
+    [bar_embed] = bar.widget_list
+    assert bar_embed.embedded == []
+    assert bar_embed.attrib == [(None, 4), ("b", 3), (None, 1), ("i", 3)]
+    assert bar_embed.text == "foo bar baz"

toot/output.py

@@ -6,7 +6,7 @@ import textwrap
 from functools import lru_cache
 from toot import settings
 from toot.entities import Instance, Notification, Poll, Status
-from toot.utils import get_text, parse_html
+from toot.utils import get_text, html_to_paragraphs
 from toot.wcstring import wc_wrap
 from typing import List
 from wcwidth import wcswidth
@@ -321,7 +321,7 @@ def print_status(status: Status, width: int = 80):
 
 def print_html(text, width=80):
     first = True
-    for paragraph in parse_html(text):
+    for paragraph in html_to_paragraphs(text):
         if not first:
             print_out("")
         for line in paragraph:

toot/tui/overlays.py

@@ -7,7 +7,7 @@ from toot import __version__
 from toot import api
 from toot.tui.utils import highlight_keys
 from toot.tui.widgets import Button, EditBox, SelectableText
-from toot.tui.richtext import ContentParser
+from toot.tui.richtext import html_to_widgets
 
 
 class StatusSource(urwid.Padding):
@@ -255,8 +255,6 @@ class Account(urwid.ListBox):
         super().__init__(walker)
 
     def generate_contents(self, account, relationship=None, last_action=None):
-        parser = ContentParser()
-
         if self.last_action and not self.last_action.startswith("Confirm"):
             yield Button(f"Confirm {self.last_action}", on_press=take_action, user_data=self)
             yield Button("Cancel", on_press=cancel_action, user_data=self)
@@ -282,7 +280,7 @@ class Account(urwid.ListBox):
         if account["note"]:
             yield urwid.Divider()
 
-            widgetlist = parser.html_to_widgets(account["note"])
+            widgetlist = html_to_widgets(account["note"])
             for line in widgetlist:
                 yield (line)
 
@@ -317,7 +315,7 @@ class Account(urwid.ListBox):
                 yield urwid.Divider()
                 yield urwid.Text([("bold", f"{name.rstrip(':')}"), ":"])
 
-                widgetlist = parser.html_to_widgets(field["value"])
+                widgetlist = html_to_widgets(field["value"])
                 for line in widgetlist:
                     yield (line)
 

toot/tui/poll.py

@@ -4,7 +4,7 @@ from toot import api
 from toot.exceptions import ApiError
 from toot.utils.datetime import parse_datetime
 from .widgets import Button, CheckBox, RadioButton
-from .richtext import ContentParser
+from .richtext import html_to_widgets
 
 
 class Poll(urwid.ListBox):
@@ -86,8 +86,7 @@ class Poll(urwid.ListBox):
     def generate_contents(self, status):
         yield urwid.Divider()
 
-        parser = ContentParser()
+        widgetlist = html_to_widgets(status.data["content"])
-        widgetlist = parser.html_to_widgets(status.data["content"])
 
         for line in widgetlist:
             yield (line)

toot/tui/richtext.py

@@ -1,457 +0,0 @@
-"""
-richtext
-"""
-from typing import List, Tuple
-import re
-import urwid
-import unicodedata
-from .constants import PALETTE
-from bs4 import BeautifulSoup
-from bs4.element import NavigableString, Tag
-from .stubs.urwidgets import TextEmbed, Hyperlink, parse_text, has_urwidgets
-from urwid.util import decompose_tagmarkup
-from toot.utils import urlencode_url
-
-
-class ContentParser:
-    def __init__(self):
-        self.palette_names = []
-        for p in PALETTE:
-            self.palette_names.append(p[0])
-
-        """Parse a limited subset of HTML and create urwid widgets."""
-
-    def html_to_widgets(self, html, recovery_attempt=False) -> List[urwid.Widget]:
-        """Convert html to urwid widgets"""
-        widgets: List[urwid.Widget] = []
-        html = unicodedata.normalize("NFKC", html)
-        soup = BeautifulSoup(html.replace("'", "'"), "html.parser")
-        first_tag = True
-        for e in soup.body or soup:
-            if isinstance(e, NavigableString):
-                if first_tag and not recovery_attempt:
-                    # if our first "tag" is a navigable string
-                    # the HTML is out of spec, doesn't start with a tag,
-                    # we see this in content from Pixelfed servers.
-                    # attempt a fix by wrapping the HTML with <p></p>
-                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
-                else:
-                    continue
-            else:
-                name = e.name
-                # if our HTML starts with a tag, but not a block tag
-                # the HTML is out of spec. Attempt a fix by wrapping the
-                # HTML with <p></p>
-                if (
-                    first_tag
-                    and not recovery_attempt
-                    and name
-                    not in (
-                        "p",
-                        "pre",
-                        "li",
-                        "blockquote",
-                        "h1",
-                        "h2",
-                        "h3",
-                        "h4",
-                        "h5",
-                        "h6",
-                    )  # NOTE: update this list if Mastodon starts supporting more block tags
-                ):
-                    return self.html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
-
-                # First, look for a custom tag handler method in this class
-                # If that fails, fall back to inline_tag_to_text handler
-                method = getattr(self, "_" + name, self.inline_tag_to_text)
-                markup = method(e)  # either returns a Widget, or plain text
-                first_tag = False
-
-            if not isinstance(markup, urwid.Widget):
-                # plaintext, so create a padded text widget
-                txt = self.text_to_widget("", markup)
-                markup = urwid.Padding(
-                    txt,
-                    align="left",
-                    width=("relative", 100),
-                    min_width=None,
-                )
-            widgets.append(markup)
-            # separate top level widgets with a blank line
-            widgets.append(urwid.Divider(" "))
-        return widgets[:-1]  # but suppress the last blank line
-
-    def inline_tag_to_text(self, tag) -> Tuple:
-        """Convert html tag to plain text with tag as attributes recursively"""
-        markups = self.process_inline_tag_children(tag)
-        if not markups:
-            return (tag.name, "")
-        return (tag.name, markups)
-
-    def process_inline_tag_children(self, tag) -> List:
-        """Recursively retrieve all children
-        and convert to a list of markup text"""
-        markups = []
-        for child in tag.children:
-            if isinstance(child, Tag):
-                method = getattr(self, "_" + child.name, self.inline_tag_to_text)
-                markup = method(child)
-                markups.append(markup)
-            else:
-                markups.append(child)
-        return markups
-
-    def text_to_widget(self, attr, markup) -> urwid.Widget:
-        if not has_urwidgets:
-            return urwid.Text((attr, markup))
-
-        TRANSFORM = {
-            # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
-            re.compile(r"(^.+)\x03(.+$)"): lambda g: (
-                len(g[1]),
-                urwid.Filler(Hyperlink(g[2], anchor_attr, g[1])),
-            ),
-        }
-        markup_list = []
-
-        for run in markup:
-            if isinstance(run, tuple):
-                txt, attr_list = decompose_tagmarkup(run)
-                # find anchor titles with an ETX separator followed by href
-                m = re.match(r"(^.+)\x03(.+$)", txt)
-                if m:
-                    anchor_attr = self.get_best_anchor_attr(attr_list)
-                    markup_list.append(
-                        parse_text(
-                            txt,
-                            TRANSFORM,
-                            lambda pattern, groups, span: TRANSFORM[pattern](groups),
-                        )
-                    )
-                else:
-                    markup_list.append(run)
-            else:
-                markup_list.append(run)
-
-        return TextEmbed(markup_list)
-
-    def process_block_tag_children(self, tag) -> List[urwid.Widget]:
-        """Recursively retrieve all children
-        and convert to a list of widgets
-        any inline tags containing text will be
-        converted to Text widgets"""
-
-        pre_widget_markups = []
-        post_widget_markups = []
-        child_widgets = []
-        found_nested_widget = False
-
-        for child in tag.children:
-            if isinstance(child, Tag):
-                # child is a nested tag; process using custom method
-                # or default to inline_tag_to_text
-                method = getattr(self, "_" + child.name, self.inline_tag_to_text)
-                result = method(child)
-                if isinstance(result, urwid.Widget):
-                    found_nested_widget = True
-                    child_widgets.append(result)
-                else:
-                    if not found_nested_widget:
-                        pre_widget_markups.append(result)
-                    else:
-                        post_widget_markups.append(result)
-            else:
-                # child is text; append to the appropriate markup list
-                if not found_nested_widget:
-                    pre_widget_markups.append(child)
-                else:
-                    post_widget_markups.append(child)
-
-        widget_list = []
-        if len(pre_widget_markups):
-            widget_list.append(self.text_to_widget(tag.name, pre_widget_markups))
-
-        if len(child_widgets):
-            widget_list += child_widgets
-
-        if len(post_widget_markups):
-            widget_list.append(self.text_to_widget(tag.name, post_widget_markups))
-
-        return widget_list
-
-    def get_urwid_attr_name(self, tag) -> str:
-        """Get the class name and translate to a
-        name suitable for use as an urwid
-        text attribute name"""
-
-        if "class" in tag.attrs:
-            clss = tag.attrs["class"]
-            if len(clss) > 0:
-                style_name = "class_" + "_".join(clss)
-                # return the class name, only if we
-                # find it as a defined palette name
-                if style_name in self.palette_names:
-                    return style_name
-
-        # fallback to returning the tag name
-        return tag.name
-
-    # Tag handlers start here.
-    # Tags not explicitly listed are "supported" by
-    # rendering as text.
-    # Inline tags return a list of marked up text for urwid.Text
-    # Block tags return urwid.Widget
-
-    def basic_block_tag_handler(self, tag) -> urwid.Widget:
-        """default for block tags that need no special treatment"""
-        return urwid.Pile(self.process_block_tag_children(tag))
-
-    def get_best_anchor_attr(self, attrib_list) -> str:
-        if not attrib_list:
-            return ""
-        flat_al = list(flatten(attrib_list))
-
-        for a in flat_al[0]:
-            # ref: https://docs.joinmastodon.org/spec/activitypub/
-            # these are the class names (translated to attrib names)
-            # that we can support for display
-
-            try:
-                if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]:
-                    return a[0]
-            except KeyError:
-                continue
-
-        return "a"
-
-    def _a(self, tag) -> Tuple:
-        """anchor tag handler"""
-
-        markups = self.process_inline_tag_children(tag)
-        if not markups:
-            return (tag.name, "")
-
-        href = tag.attrs["href"]
-        title, attrib_list = decompose_tagmarkup(markups)
-        if not attrib_list:
-            attrib_list = [tag]
-        if href and has_urwidgets:
-            # only if we have urwidgets loaded for OCS 8 hyperlinks:
-            # urlencode the path and query portions of the URL
-            href = urlencode_url(href)
-            # use ASCII ETX (end of record) as a
-            # delimiter between the title and the HREF
-            title += f"\x03{href}"
-
-        attr = self.get_best_anchor_attr(attrib_list)
-
-        if attr == "a":
-            # didn't find an attribute to use
-            # in the child markup, so let's
-            # try the anchor tag's own attributes
-
-            attr = self.get_urwid_attr_name(tag)
-
-        # hashtag anchors have a class of "mention hashtag"
-        # or "hashtag"
-        # we'll return style "class_mention_hashtag"
-        # or "class_hashtag"
-        # in that case; see corresponding palette entry
-        # in constants.py controlling hashtag highlighting
-
-        return (attr, title)
-
-    def _blockquote(self, tag) -> urwid.Widget:
-        widget_list = self.process_block_tag_children(tag)
-        blockquote_widget = urwid.LineBox(
-            urwid.Padding(
-                urwid.Pile(widget_list),
-                align="left",
-                width=("relative", 100),
-                min_width=None,
-                left=1,
-                right=1,
-            ),
-            tlcorner="",
-            tline="",
-            lline="│",
-            trcorner="",
-            blcorner="",
-            rline="",
-            bline="",
-            brcorner="",
-        )
-        return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
-
-    def _br(self, tag) -> Tuple:
-        return ("br", "\n")
-
-    def _em(self, tag) -> Tuple:
-        # to simplify the number of palette entries
-        # translate EM to I (italic)
-        markups = self.process_inline_tag_children(tag)
-        if not markups:
-            return ("i", "")
-
-        # special case processing for bold and italic
-        for parent in tag.parents:
-            if parent.name == "b" or parent.name == "strong":
-                return ("bi", markups)
-
-        return ("i", markups)
-
-    def _ol(self, tag) -> urwid.Widget:
-        """ordered list tag handler"""
-
-        widgets = []
-        list_item_num = 1
-        increment = -1 if tag.has_attr("reversed") else 1
-
-        # get ol start= attribute if present
-        if tag.has_attr("start") and len(tag.attrs["start"]) > 0:
-            try:
-                list_item_num = int(tag.attrs["start"])
-            except ValueError:
-                pass
-
-        for li in tag.find_all("li", recursive=False):
-            method = getattr(self, "_li", self.inline_tag_to_text)
-            markup = method(li)
-
-            # li value= attribute will change the item number
-            # it also overrides any ol start= attribute
-
-            if li.has_attr("value") and len(li.attrs["value"]) > 0:
-                try:
-                    list_item_num = int(li.attrs["value"])
-                except ValueError:
-                    pass
-
-            if not isinstance(markup, urwid.Widget):
-                txt = self.text_to_widget("li", [str(list_item_num), ". ", markup])
-                # 1. foo, 2. bar, etc.
-                widgets.append(txt)
-            else:
-                txt = self.text_to_widget("li", [str(list_item_num), ". "])
-                columns = urwid.Columns(
-                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
-                )
-                widgets.append(columns)
-
-            list_item_num += increment
-
-        return urwid.Pile(widgets)
-
-    def _pre(self, tag) -> urwid.Widget:
-        # <PRE> tag spec says that text should not wrap,
-        # but horizontal screen space is at a premium
-        # and we have no horizontal scroll bar, so allow
-        # wrapping.
-
-        widget_list = [urwid.Divider(" ")]
-        widget_list += self.process_block_tag_children(tag)
-
-        pre_widget = urwid.Padding(
-            urwid.Pile(widget_list),
-            align="left",
-            width=("relative", 100),
-            min_width=None,
-            left=1,
-            right=1,
-        )
-        return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
-
-    def _span(self, tag) -> Tuple:
-        markups = self.process_inline_tag_children(tag)
-
-        if not markups:
-            return (tag.name, "")
-
-        # span inherits its parent's class definition
-        # unless it has a specific class definition
-        # of its own
-
-        if "class" in tag.attrs:
-            # uncomment the following code to hide all HTML marked
-            # invisible (generally, the http:// prefix of URLs)
-            # could be a user preference, it's only advisable if
-            # the terminal supports OCS 8 hyperlinks (and that's not
-            # automatically detectable)
-
-            # if "invisible" in tag.attrs["class"]:
-            #     return (tag.name, "")
-
-            style_name = self.get_urwid_attr_name(tag)
-
-            if style_name != "span":
-                # unique class name matches an entry in our palette
-                return (style_name, markups)
-
-        if tag.parent:
-            return (self.get_urwid_attr_name(tag.parent), markups)
-        else:
-            # fallback
-            return ("span", markups)
-
-    def _strong(self, tag) -> Tuple:
-        # to simplify the number of palette entries
-        # translate STRONG to B (bold)
-        markups = self.process_inline_tag_children(tag)
-        if not markups:
-            return ("b", "")
-
-        # special case processing for bold and italic
-        for parent in tag.parents:
-            if parent.name == "i" or parent.name == "em":
-                return ("bi", markups)
-
-        return ("b", markups)
-
-    def _ul(self, tag) -> urwid.Widget:
-        """unordered list tag handler"""
-
-        widgets = []
-
-        for li in tag.find_all("li", recursive=False):
-            method = getattr(self, "_li", self.inline_tag_to_text)
-            markup = method(li)
-
-            if not isinstance(markup, urwid.Widget):
-                txt = self.text_to_widget("li", ["\N{bullet} ", markup])
-                # * foo, * bar, etc.
-                widgets.append(txt)
-            else:
-                txt = self.text_to_widget("li", ["\N{bullet} "])
-                columns = urwid.Columns(
-                    [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
-                )
-                widgets.append(columns)
-
-        return urwid.Pile(widgets)
-
-    # These tags are handled identically to others
-    # the only difference being the tag name used for
-    # urwid attribute mapping
-
-    _b = _strong
-
-    _div = basic_block_tag_handler
-
-    _i = _em
-
-    _li = basic_block_tag_handler
-
-    # Glitch-soc and Pleroma allow <H1>...<H6> in content
-    # Mastodon (PR #23913) does not; header tags are converted to <P><STRONG></STRONG></P>
-
-    _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
-
-    _p = basic_block_tag_handler
-
-
-def flatten(data):
-    if isinstance(data, tuple):
-        for x in data:
-            yield from flatten(x)
-    else:
-        yield data

toot/tui/richtext/__init__.py

@@ -0,0 +1,18 @@
+import urwid
+
+from toot.tui.utils import highlight_hashtags
+from toot.utils import format_content
+from typing import List
+
+try:
+    from .richtext import html_to_widgets, url_to_widget
+except ImportError:
+    # Fallback if urwidgets are not available
+    def html_to_widgets(html: str) -> List[urwid.Widget]:
+        return [
+            urwid.Text(highlight_hashtags(line))
+            for line in format_content(html)
+        ]
+
+    def url_to_widget(url: str):
+        return urwid.Text(("link", url))

toot/tui/richtext/richtext.py

@@ -0,0 +1,452 @@
+import re
+import urwid
+import unicodedata
+
+from bs4.element import NavigableString, Tag
+from toot.tui.constants import PALETTE
+from toot.utils import parse_html, urlencode_url
+from typing import List, Tuple
+from urwid.util import decompose_tagmarkup
+from urwidgets import Hyperlink, TextEmbed
+
+
+STYLE_NAMES = [p[0] for p in PALETTE]
+
+# NOTE: update this list if Mastodon starts supporting more block tags
+BLOCK_TAGS = ["p", "pre", "li", "blockquote", "h1", "h2", "h3", "h4", "h5", "h6"]
+
+
+def html_to_widgets(html, recovery_attempt=False) -> List[urwid.Widget]:
+    """Convert html to urwid widgets"""
+    widgets: List[urwid.Widget] = []
+    html = unicodedata.normalize("NFKC", html)
+    soup = parse_html(html)
+
+    first_tag = True
+    for e in soup.body or soup:
+        if isinstance(e, NavigableString):
+            if first_tag and not recovery_attempt:
+                # if our first "tag" is a navigable string
+                # the HTML is out of spec, doesn't start with a tag,
+                # we see this in content from Pixelfed servers.
+                # attempt a fix by wrapping the HTML with <p></p>
+                return html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
+            else:
+                continue
+        else:
+            name = e.name
+            # if our HTML starts with a tag, but not a block tag
+            # the HTML is out of spec. Attempt a fix by wrapping the
+            # HTML with <p></p>
+            if (first_tag and not recovery_attempt and name not in BLOCK_TAGS):
+                return html_to_widgets(f"<p>{html}</p>", recovery_attempt=True)
+
+            markup = render(name, e)
+            first_tag = False
+
+        if not isinstance(markup, urwid.Widget):
+            # plaintext, so create a padded text widget
+            txt = text_to_widget("", markup)
+            markup = urwid.Padding(
+                txt,
+                align="left",
+                width=("relative", 100),
+                min_width=None,
+            )
+        widgets.append(markup)
+        # separate top level widgets with a blank line
+        widgets.append(urwid.Divider(" "))
+    return widgets[:-1]  # but suppress the last blank line
+
+
+def url_to_widget(url: str):
+    widget = len(url), urwid.Filler(Hyperlink(url, "link", url))
+    return TextEmbed(widget)
+
+
+def inline_tag_to_text(tag) -> Tuple:
+    """Convert html tag to plain text with tag as attributes recursively"""
+    markups = process_inline_tag_children(tag)
+    if not markups:
+        return (tag.name, "")
+    return (tag.name, markups)
+
+
+def process_inline_tag_children(tag) -> List:
+    """Recursively retrieve all children
+    and convert to a list of markup text"""
+    markups = []
+    for child in tag.children:
+        if isinstance(child, Tag):
+            markup = render(child.name, child)
+            markups.append(markup)
+        else:
+            markups.append(child)
+    return markups
+
+
+URL_PATTERN = re.compile(r"(^.+)\x03(.+$)")
+
+
+def text_to_widget(attr, markup) -> urwid.Widget:
+    markup_list = []
+    for run in markup:
+        if isinstance(run, tuple):
+            txt, attr_list = decompose_tagmarkup(run)
+            # find anchor titles with an ETX separator followed by href
+            match = URL_PATTERN.match(txt)
+            if match:
+                label, url = match.groups()
+                anchor_attr = get_best_anchor_attr(attr_list)
+                markup_list.append((
+                    len(label),
+                    urwid.Filler(Hyperlink(url, anchor_attr, label)),
+                ))
+            else:
+                markup_list.append(run)
+        else:
+            markup_list.append(run)
+
+    return TextEmbed(markup_list)
+
+
+def process_block_tag_children(tag) -> List[urwid.Widget]:
+    """Recursively retrieve all children
+    and convert to a list of widgets
+    any inline tags containing text will be
+    converted to Text widgets"""
+
+    pre_widget_markups = []
+    post_widget_markups = []
+    child_widgets = []
+    found_nested_widget = False
+
+    for child in tag.children:
+        if isinstance(child, Tag):
+            # child is a nested tag; process using custom method
+            # or default to inline_tag_to_text
+            result = render(child.name, child)
+            if isinstance(result, urwid.Widget):
+                found_nested_widget = True
+                child_widgets.append(result)
+            else:
+                if not found_nested_widget:
+                    pre_widget_markups.append(result)
+                else:
+                    post_widget_markups.append(result)
+        else:
+            # child is text; append to the appropriate markup list
+            if not found_nested_widget:
+                pre_widget_markups.append(child)
+            else:
+                post_widget_markups.append(child)
+
+    widget_list = []
+    if len(pre_widget_markups):
+        widget_list.append(text_to_widget(tag.name, pre_widget_markups))
+
+    if len(child_widgets):
+        widget_list += child_widgets
+
+    if len(post_widget_markups):
+        widget_list.append(text_to_widget(tag.name, post_widget_markups))
+
+    return widget_list
+
+
+def get_urwid_attr_name(tag) -> str:
+    """Get the class name and translate to a
+    name suitable for use as an urwid
+    text attribute name"""
+
+    if "class" in tag.attrs:
+        clss = tag.attrs["class"]
+        if len(clss) > 0:
+            style_name = "class_" + "_".join(clss)
+            # return the class name, only if we
+            # find it as a defined palette name
+            if style_name in STYLE_NAMES:
+                return style_name
+
+    # fallback to returning the tag name
+    return tag.name
+
+
+def basic_block_tag_handler(tag) -> urwid.Widget:
+    """default for block tags that need no special treatment"""
+    return urwid.Pile(process_block_tag_children(tag))
+
+
+def get_best_anchor_attr(attrib_list) -> str:
+    if not attrib_list:
+        return ""
+    flat_al = list(flatten(attrib_list))
+
+    for a in flat_al[0]:
+        # ref: https://docs.joinmastodon.org/spec/activitypub/
+        # these are the class names (translated to attrib names)
+        # that we can support for display
+
+        try:
+            if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]:
+                return a[0]
+        except KeyError:
+            continue
+
+    return "a"
+
+
+def render(attr: str, content: str):
+    if attr in ["a"]:
+        return render_anchor(content)
+
+    if attr in ["blockquote"]:
+        return render_blockquote(content)
+
+    if attr in ["br"]:
+        return render_br(content)
+
+    if attr in ["em"]:
+        return render_em(content)
+
+    if attr in ["ol"]:
+        return render_ol(content)
+
+    if attr in ["pre"]:
+        return render_pre(content)
+
+    if attr in ["span"]:
+        return render_span(content)
+
+    if attr in ["b", "strong"]:
+        return render_strong(content)
+
+    if attr in ["ul"]:
+        return render_ul(content)
+
+    # Glitch-soc and Pleroma allow <H1>...<H6> in content
+    # Mastodon (PR #23913) does not; header tags are converted to <P><STRONG></STRONG></P>
+    if attr in ["p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6"]:
+        return basic_block_tag_handler(content)
+
+    # Fall back to inline_tag_to_text handler
+    return inline_tag_to_text(content)
+
+
+def render_anchor(tag) -> Tuple:
+    """anchor tag handler"""
+
+    markups = process_inline_tag_children(tag)
+    if not markups:
+        return (tag.name, "")
+
+    href = tag.attrs["href"]
+    title, attrib_list = decompose_tagmarkup(markups)
+    if not attrib_list:
+        attrib_list = [tag]
+    if href:
+        # urlencode the path and query portions of the URL
+        href = urlencode_url(href)
+        # use ASCII ETX (end of record) as a
+        # delimiter between the title and the HREF
+        title += f"\x03{href}"
+
+    attr = get_best_anchor_attr(attrib_list)
+
+    if attr == "a":
+        # didn't find an attribute to use
+        # in the child markup, so let's
+        # try the anchor tag's own attributes
+
+        attr = get_urwid_attr_name(tag)
+
+    # hashtag anchors have a class of "mention hashtag"
+    # or "hashtag"
+    # we'll return style "class_mention_hashtag"
+    # or "class_hashtag"
+    # in that case; see corresponding palette entry
+    # in constants.py controlling hashtag highlighting
+
+    return (attr, title)
+
+
+def render_blockquote(tag) -> urwid.Widget:
+    widget_list = process_block_tag_children(tag)
+    blockquote_widget = urwid.LineBox(
+        urwid.Padding(
+            urwid.Pile(widget_list),
+            align="left",
+            width=("relative", 100),
+            min_width=None,
+            left=1,
+            right=1,
+        ),
+        tlcorner="",
+        tline="",
+        lline="│",
+        trcorner="",
+        blcorner="",
+        rline="",
+        bline="",
+        brcorner="",
+    )
+    return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
+
+
+def render_br(tag) -> Tuple:
+    return ("br", "\n")
+
+
+def render_em(tag) -> Tuple:
+    # to simplify the number of palette entries
+    # translate EM to I (italic)
+    markups = process_inline_tag_children(tag)
+    if not markups:
+        return ("i", "")
+
+    # special case processing for bold and italic
+    for parent in tag.parents:
+        if parent.name == "b" or parent.name == "strong":
+            return ("bi", markups)
+
+    return ("i", markups)
+
+
+def render_ol(tag) -> urwid.Widget:
+    """ordered list tag handler"""
+
+    widgets = []
+    list_item_num = 1
+    increment = -1 if tag.has_attr("reversed") else 1
+
+    # get ol start= attribute if present
+    if tag.has_attr("start") and len(tag.attrs["start"]) > 0:
+        try:
+            list_item_num = int(tag.attrs["start"])
+        except ValueError:
+            pass
+
+    for li in tag.find_all("li", recursive=False):
+        markup = render("li", li)
+
+        # li value= attribute will change the item number
+        # it also overrides any ol start= attribute
+
+        if li.has_attr("value") and len(li.attrs["value"]) > 0:
+            try:
+                list_item_num = int(li.attrs["value"])
+            except ValueError:
+                pass
+
+        if not isinstance(markup, urwid.Widget):
+            txt = text_to_widget("li", [str(list_item_num), ". ", markup])
+            # 1. foo, 2. bar, etc.
+            widgets.append(txt)
+        else:
+            txt = text_to_widget("li", [str(list_item_num), ". "])
+            columns = urwid.Columns(
+                [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+            )
+            widgets.append(columns)
+
+        list_item_num += increment
+
+    return urwid.Pile(widgets)
+
+
+def render_pre(tag) -> urwid.Widget:
+    # <PRE> tag spec says that text should not wrap,
+    # but horizontal screen space is at a premium
+    # and we have no horizontal scroll bar, so allow
+    # wrapping.
+
+    widget_list = [urwid.Divider(" ")]
+    widget_list += process_block_tag_children(tag)
+
+    pre_widget = urwid.Padding(
+        urwid.Pile(widget_list),
+        align="left",
+        width=("relative", 100),
+        min_width=None,
+        left=1,
+        right=1,
+    )
+    return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
+
+
+def render_span(tag) -> Tuple:
+    markups = process_inline_tag_children(tag)
+
+    if not markups:
+        return (tag.name, "")
+
+    # span inherits its parent's class definition
+    # unless it has a specific class definition
+    # of its own
+
+    if "class" in tag.attrs:
+        # uncomment the following code to hide all HTML marked
+        # invisible (generally, the http:// prefix of URLs)
+        # could be a user preference, it's only advisable if
+        # the terminal supports OCS 8 hyperlinks (and that's not
+        # automatically detectable)
+
+        # if "invisible" in tag.attrs["class"]:
+        #     return (tag.name, "")
+
+        style_name = get_urwid_attr_name(tag)
+
+        if style_name != "span":
+            # unique class name matches an entry in our palette
+            return (style_name, markups)
+
+    if tag.parent:
+        return (get_urwid_attr_name(tag.parent), markups)
+    else:
+        # fallback
+        return ("span", markups)
+
+
+def render_strong(tag) -> Tuple:
+    # to simplify the number of palette entries
+    # translate STRONG to B (bold)
+    markups = process_inline_tag_children(tag)
+    if not markups:
+        return ("b", "")
+
+    # special case processing for bold and italic
+    for parent in tag.parents:
+        if parent.name == "i" or parent.name == "em":
+            return ("bi", markups)
+
+    return ("b", markups)
+
+
+def render_ul(tag) -> urwid.Widget:
+    """unordered list tag handler"""
+
+    widgets = []
+
+    for li in tag.find_all("li", recursive=False):
+        markup = render("li", li)
+
+        if not isinstance(markup, urwid.Widget):
+            txt = text_to_widget("li", ["\N{bullet} ", markup])
+            # * foo, * bar, etc.
+            widgets.append(txt)
+        else:
+            txt = text_to_widget("li", ["\N{bullet} "])
+            columns = urwid.Columns(
+                [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+            )
+            widgets.append(columns)
+
+    return urwid.Pile(widgets)
+
+
+def flatten(data):
+    if isinstance(data, tuple):
+        for x in data:
+            yield from flatten(x)
+    else:
+        yield data

toot/tui/stubs/stub_hyperlink.py

@@ -1,30 +0,0 @@
-__all__ = ("Hyperlink",)
-
-import urwid
-
-
-class Hyperlink(urwid.WidgetWrap):
-    def __init__(
-        self,
-        uri,
-        attr,
-        text,
-    ):
-        pass
-
-    def render(self, size, focus):
-        return None
-
-
-class HyperlinkCanvas(urwid.Canvas):
-    def __init__(self, uri: str, text_canv: urwid.TextCanvas):
-        pass
-
-    def cols(self):
-        return 0
-
-    def content(self, *args, **kwargs):
-        yield [None]
-
-    def rows(self):
-        return 0

toot/tui/stubs/stub_text_embed.py

@@ -1,29 +0,0 @@
-__all__ = ("parse_text", "TextEmbed")
-
-import urwid
-
-
-class TextEmbed(urwid.Text):
-    def get_text(
-        self,
-    ):
-        return None
-
-    def render(self, size, focus):
-        return None
-
-    def set_text(self, markup):
-        pass
-
-    def set_wrap_mode(self, mode):
-        pass
-
-
-def parse_text(
-    text,
-    patterns,
-    repl,
-    *repl_args,
-    **repl_kwargs,
-):
-    return None

toot/tui/stubs/urwidgets.py

@@ -1,8 +0,0 @@
-# If urwidgets is loaded use it; otherwise use our stubs
-try:
-    from urwidgets import Hyperlink, TextEmbed, parse_text
-    has_urwidgets = True
-except ImportError:
-    from .stub_hyperlink import Hyperlink
-    from .stub_text_embed import TextEmbed, parse_text
-    has_urwidgets = False

toot/tui/timeline.py

@@ -1,11 +1,11 @@
 import logging
-import re
 import urwid
 import webbrowser
 
 from typing import List, Optional
 
 from toot.tui import app
+from toot.tui.richtext import html_to_widgets, url_to_widget
 from toot.utils.datetime import parse_datetime, time_ago
 from toot.utils.language import language_name
 
@@ -13,9 +13,6 @@ from toot.entities import Status
 from toot.tui.scroll import Scrollable, ScrollBar
 from toot.tui.utils import highlight_keys
 from toot.tui.widgets import SelectableText, SelectableColumns
-from toot.tui.richtext import ContentParser
-from toot.utils import urlencode_url
-from toot.tui.stubs.urwidgets import Hyperlink, TextEmbed, parse_text, has_urwidgets
 
 logger = logging.getLogger("toot")
 
@@ -320,20 +317,6 @@ class StatusDetails(urwid.Pile):
             if status else ())
         return super().__init__(widget_list)
 
-    def linkify_content(self, text) -> urwid.Widget:
-        if not has_urwidgets:
-            return urwid.Text(("link", text))
-        TRANSFORM = {
-            # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
-            re.compile(r'(https?://[^\s]+)'):
-                lambda g: (len(g[1]), urwid.Filler(Hyperlink(urlencode_url(g[1]), "link", g[1]))),
-        }
-        markup_list = []
-
-        markup_list.append(parse_text(text, TRANSFORM,
-            lambda pattern, groups, span: TRANSFORM[pattern](groups)))
-        return TextEmbed(markup_list, align='left')
-
     def content_generator(self, status, reblogged_by):
         if reblogged_by:
             text = "♺ {} boosted".format(reblogged_by.display_name or reblogged_by.username)
@@ -356,9 +339,7 @@ class StatusDetails(urwid.Pile):
             yield ("pack", urwid.Text(("content_warning", "Marked as sensitive. Press S to view.")))
         else:
             content = status.original.translation if status.original.show_translation else status.data["content"]
-
+            widgetlist = html_to_widgets(content)
-            parser = ContentParser()
-            widgetlist = parser.html_to_widgets(content)
 
             for line in widgetlist:
                 yield (line)
@@ -370,7 +351,7 @@ class StatusDetails(urwid.Pile):
                     yield ("pack", urwid.Text([("bold", "Media attachment"), " (", m["type"], ")"]))
                     if m["description"]:
                         yield ("pack", urwid.Text(m["description"]))
-                    yield ("pack", self.linkify_content(m["url"]))
+                    yield ("pack", url_to_widget(m["url"]))
 
             poll = status.original.data.get("poll")
             if poll:
@@ -430,7 +411,7 @@ class StatusDetails(urwid.Pile):
         if card["description"]:
             yield urwid.Text(card["description"].strip())
             yield urwid.Text("")
-        yield self.linkify_content(card["url"])
+        yield url_to_widget(card["url"])
 
     def poll_generator(self, poll):
         for idx, option in enumerate(poll["options"]):

toot/tui/urwidgets.py

@@ -1,8 +0,0 @@
-# If urwidgets is loaded use it; otherwise use our stubs
-try:
-    from urwidgets import Hyperlink, TextEmbed, parse_text  # noqa: F401
-    has_urwidgets = True
-except ImportError:
-    from .stub_hyperlink import Hyperlink  # noqa: F401
-    from .stub_text_embed import TextEmbed, parse_text  # noqa: F401
-    has_urwidgets = False

toot/tui/utils.py

@@ -35,6 +35,18 @@ def highlight_keys(text, high_attr, low_attr=""):
     return list(_gen())
 
 
+def highlight_hashtags(line):
+    hline = []
+
+    for p in re.split(HASHTAG_PATTERN, line):
+        if p.startswith("#"):
+            hline.append(("hashtag", p))
+        else:
+            hline.append(p)
+
+    return hline
+
+
 def show_media(paths):
     """
     Attempt to open an image viewer to show given media files.

toot/utils/__init__.py

@@ -23,20 +23,22 @@ def str_bool_nullable(b):
     return None if b is None else str_bool(b)
 
 
-def get_text(html):
+def parse_html(html: str) -> BeautifulSoup:
-    """Converts html to text, strips all tags."""
-
     # Ignore warnings made by BeautifulSoup, if passed something that looks like
     # a file (e.g. a dot which matches current dict), it will warn that the file
     # should be opened instead of passing a filename.
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
-        text = BeautifulSoup(html.replace(''', "'"), "html.parser").get_text()
+        return BeautifulSoup(html.replace("'", "'"), "html.parser")
-
-    return unicodedata.normalize('NFKC', text)
 
 
-def parse_html(html):
+def get_text(html):
+    """Converts html to text, strips all tags."""
+    text = parse_html(html).get_text()
+    return unicodedata.normalize("NFKC", text)
+
+
+def html_to_paragraphs(html):
     """Attempt to convert html to plain text while keeping line breaks.
     Returns a list of paragraphs, each being a list of lines.
     """
@@ -55,7 +57,7 @@ def format_content(content):
     Returns a generator yielding lines of content.
     """
 
-    paragraphs = parse_html(content)
+    paragraphs = html_to_paragraphs(content)
 
     first = True