foo
+foo bar baz
+ """.strip() + + [foo, divider, bar] = html_to_widgets(html) + + assert isinstance(foo, Pile) + assert isinstance(divider, Divider) + assert isinstance(bar, Pile) + + [foo_embed] = foo.widget_list + assert foo_embed.embedded == [] + assert foo_embed.attrib == [] + assert foo_embed.text == "foo" + + [bar_embed] = bar.widget_list + assert bar_embed.embedded == [] + assert bar_embed.attrib == [(None, 4), ("b", 3), (None, 1), ("i", 3)] + assert bar_embed.text == "foo bar baz" diff --git a/toot/output.py b/toot/output.py index 6fd59a2..83898f8 100644 --- a/toot/output.py +++ b/toot/output.py @@ -6,7 +6,7 @@ import textwrap from functools import lru_cache from toot import settings from toot.entities import Instance, Notification, Poll, Status -from toot.utils import get_text, parse_html +from toot.utils import get_text, html_to_paragraphs from toot.wcstring import wc_wrap from typing import List from wcwidth import wcswidth @@ -321,7 +321,7 @@ def print_status(status: Status, width: int = 80): def print_html(text, width=80): first = True - for paragraph in parse_html(text): + for paragraph in html_to_paragraphs(text): if not first: print_out("") for line in paragraph: diff --git a/toot/tui/overlays.py b/toot/tui/overlays.py index 530921a..58eb457 100644 --- a/toot/tui/overlays.py +++ b/toot/tui/overlays.py @@ -7,7 +7,7 @@ from toot import __version__ from toot import api from toot.tui.utils import highlight_keys from toot.tui.widgets import Button, EditBox, SelectableText -from toot.tui.richtext import ContentParser +from toot.tui.richtext import html_to_widgets class StatusSource(urwid.Padding): @@ -255,8 +255,6 @@ class Account(urwid.ListBox): super().__init__(walker) def generate_contents(self, account, relationship=None, last_action=None): - parser = ContentParser() - if self.last_action and not self.last_action.startswith("Confirm"): yield Button(f"Confirm {self.last_action}", on_press=take_action, user_data=self) yield Button("Cancel", on_press=cancel_action, user_data=self) @@ -282,7 +280,7 @@ class Account(urwid.ListBox): if account["note"]: yield urwid.Divider() - widgetlist = parser.html_to_widgets(account["note"]) + widgetlist = html_to_widgets(account["note"]) for line in widgetlist: yield (line) @@ -317,7 +315,7 @@ class Account(urwid.ListBox): yield urwid.Divider() yield urwid.Text([("bold", f"{name.rstrip(':')}"), ":"]) - widgetlist = parser.html_to_widgets(field["value"]) + widgetlist = html_to_widgets(field["value"]) for line in widgetlist: yield (line) diff --git a/toot/tui/poll.py b/toot/tui/poll.py index c92cc07..e738fc7 100644 --- a/toot/tui/poll.py +++ b/toot/tui/poll.py @@ -4,7 +4,7 @@ from toot import api from toot.exceptions import ApiError from toot.utils.datetime import parse_datetime from .widgets import Button, CheckBox, RadioButton -from .richtext import ContentParser +from .richtext import html_to_widgets class Poll(urwid.ListBox): @@ -86,8 +86,7 @@ class Poll(urwid.ListBox): def generate_contents(self, status): yield urwid.Divider() - parser = ContentParser() - widgetlist = parser.html_to_widgets(status.data["content"]) + widgetlist = html_to_widgets(status.data["content"]) for line in widgetlist: yield (line) diff --git a/toot/tui/richtext.py b/toot/tui/richtext.py deleted file mode 100644 index e74dff8..0000000 --- a/toot/tui/richtext.py +++ /dev/null @@ -1,457 +0,0 @@ -""" -richtext -""" -from typing import List, Tuple -import re -import urwid -import unicodedata -from .constants import PALETTE -from bs4 import BeautifulSoup -from bs4.element import NavigableString, Tag -from .stubs.urwidgets import TextEmbed, Hyperlink, parse_text, has_urwidgets -from urwid.util import decompose_tagmarkup -from toot.utils import urlencode_url - - -class ContentParser: - def __init__(self): - self.palette_names = [] - for p in PALETTE: - self.palette_names.append(p[0]) - - """Parse a limited subset of HTML and create urwid widgets.""" - - def html_to_widgets(self, html, recovery_attempt=False) -> List[urwid.Widget]: - """Convert html to urwid widgets""" - widgets: List[urwid.Widget] = [] - html = unicodedata.normalize("NFKC", html) - soup = BeautifulSoup(html.replace("'", "'"), "html.parser") - first_tag = True - for e in soup.body or soup: - if isinstance(e, NavigableString): - if first_tag and not recovery_attempt: - # if our first "tag" is a navigable string - # the HTML is out of spec, doesn't start with a tag, - # we see this in content from Pixelfed servers. - # attempt a fix by wrapping the HTML with - return self.html_to_widgets(f"{html}
", recovery_attempt=True) - else: - continue - else: - name = e.name - # if our HTML starts with a tag, but not a block tag - # the HTML is out of spec. Attempt a fix by wrapping the - # HTML with - if ( - first_tag - and not recovery_attempt - and name - not in ( - "p", - "pre", - "li", - "blockquote", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - ) # NOTE: update this list if Mastodon starts supporting more block tags - ): - return self.html_to_widgets(f"{html}
", recovery_attempt=True) - - # First, look for a custom tag handler method in this class - # If that fails, fall back to inline_tag_to_text handler - method = getattr(self, "_" + name, self.inline_tag_to_text) - markup = method(e) # either returns a Widget, or plain text - first_tag = False - - if not isinstance(markup, urwid.Widget): - # plaintext, so create a padded text widget - txt = self.text_to_widget("", markup) - markup = urwid.Padding( - txt, - align="left", - width=("relative", 100), - min_width=None, - ) - widgets.append(markup) - # separate top level widgets with a blank line - widgets.append(urwid.Divider(" ")) - return widgets[:-1] # but suppress the last blank line - - def inline_tag_to_text(self, tag) -> Tuple: - """Convert html tag to plain text with tag as attributes recursively""" - markups = self.process_inline_tag_children(tag) - if not markups: - return (tag.name, "") - return (tag.name, markups) - - def process_inline_tag_children(self, tag) -> List: - """Recursively retrieve all children - and convert to a list of markup text""" - markups = [] - for child in tag.children: - if isinstance(child, Tag): - method = getattr(self, "_" + child.name, self.inline_tag_to_text) - markup = method(child) - markups.append(markup) - else: - markups.append(child) - return markups - - def text_to_widget(self, attr, markup) -> urwid.Widget: - if not has_urwidgets: - return urwid.Text((attr, markup)) - - TRANSFORM = { - # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget - re.compile(r"(^.+)\x03(.+$)"): lambda g: ( - len(g[1]), - urwid.Filler(Hyperlink(g[2], anchor_attr, g[1])), - ), - } - markup_list = [] - - for run in markup: - if isinstance(run, tuple): - txt, attr_list = decompose_tagmarkup(run) - # find anchor titles with an ETX separator followed by href - m = re.match(r"(^.+)\x03(.+$)", txt) - if m: - anchor_attr = self.get_best_anchor_attr(attr_list) - markup_list.append( - parse_text( - txt, - TRANSFORM, - lambda pattern, groups, span: TRANSFORM[pattern](groups), - ) - ) - else: - markup_list.append(run) - else: - markup_list.append(run) - - return TextEmbed(markup_list) - - def process_block_tag_children(self, tag) -> List[urwid.Widget]: - """Recursively retrieve all children - and convert to a list of widgets - any inline tags containing text will be - converted to Text widgets""" - - pre_widget_markups = [] - post_widget_markups = [] - child_widgets = [] - found_nested_widget = False - - for child in tag.children: - if isinstance(child, Tag): - # child is a nested tag; process using custom method - # or default to inline_tag_to_text - method = getattr(self, "_" + child.name, self.inline_tag_to_text) - result = method(child) - if isinstance(result, urwid.Widget): - found_nested_widget = True - child_widgets.append(result) - else: - if not found_nested_widget: - pre_widget_markups.append(result) - else: - post_widget_markups.append(result) - else: - # child is text; append to the appropriate markup list - if not found_nested_widget: - pre_widget_markups.append(child) - else: - post_widget_markups.append(child) - - widget_list = [] - if len(pre_widget_markups): - widget_list.append(self.text_to_widget(tag.name, pre_widget_markups)) - - if len(child_widgets): - widget_list += child_widgets - - if len(post_widget_markups): - widget_list.append(self.text_to_widget(tag.name, post_widget_markups)) - - return widget_list - - def get_urwid_attr_name(self, tag) -> str: - """Get the class name and translate to a - name suitable for use as an urwid - text attribute name""" - - if "class" in tag.attrs: - clss = tag.attrs["class"] - if len(clss) > 0: - style_name = "class_" + "_".join(clss) - # return the class name, only if we - # find it as a defined palette name - if style_name in self.palette_names: - return style_name - - # fallback to returning the tag name - return tag.name - - # Tag handlers start here. - # Tags not explicitly listed are "supported" by - # rendering as text. - # Inline tags return a list of marked up text for urwid.Text - # Block tags return urwid.Widget - - def basic_block_tag_handler(self, tag) -> urwid.Widget: - """default for block tags that need no special treatment""" - return urwid.Pile(self.process_block_tag_children(tag)) - - def get_best_anchor_attr(self, attrib_list) -> str: - if not attrib_list: - return "" - flat_al = list(flatten(attrib_list)) - - for a in flat_al[0]: - # ref: https://docs.joinmastodon.org/spec/activitypub/ - # these are the class names (translated to attrib names) - # that we can support for display - - try: - if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]: - return a[0] - except KeyError: - continue - - return "a" - - def _a(self, tag) -> Tuple: - """anchor tag handler""" - - markups = self.process_inline_tag_children(tag) - if not markups: - return (tag.name, "") - - href = tag.attrs["href"] - title, attrib_list = decompose_tagmarkup(markups) - if not attrib_list: - attrib_list = [tag] - if href and has_urwidgets: - # only if we have urwidgets loaded for OCS 8 hyperlinks: - # urlencode the path and query portions of the URL - href = urlencode_url(href) - # use ASCII ETX (end of record) as a - # delimiter between the title and the HREF - title += f"\x03{href}" - - attr = self.get_best_anchor_attr(attrib_list) - - if attr == "a": - # didn't find an attribute to use - # in the child markup, so let's - # try the anchor tag's own attributes - - attr = self.get_urwid_attr_name(tag) - - # hashtag anchors have a class of "mention hashtag" - # or "hashtag" - # we'll return style "class_mention_hashtag" - # or "class_hashtag" - # in that case; see corresponding palette entry - # in constants.py controlling hashtag highlighting - - return (attr, title) - - def _blockquote(self, tag) -> urwid.Widget: - widget_list = self.process_block_tag_children(tag) - blockquote_widget = urwid.LineBox( - urwid.Padding( - urwid.Pile(widget_list), - align="left", - width=("relative", 100), - min_width=None, - left=1, - right=1, - ), - tlcorner="", - tline="", - lline="│", - trcorner="", - blcorner="", - rline="", - bline="", - brcorner="", - ) - return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")]) - - def _br(self, tag) -> Tuple: - return ("br", "\n") - - def _em(self, tag) -> Tuple: - # to simplify the number of palette entries - # translate EM to I (italic) - markups = self.process_inline_tag_children(tag) - if not markups: - return ("i", "") - - # special case processing for bold and italic - for parent in tag.parents: - if parent.name == "b" or parent.name == "strong": - return ("bi", markups) - - return ("i", markups) - - def _ol(self, tag) -> urwid.Widget: - """ordered list tag handler""" - - widgets = [] - list_item_num = 1 - increment = -1 if tag.has_attr("reversed") else 1 - - # get ol start= attribute if present - if tag.has_attr("start") and len(tag.attrs["start"]) > 0: - try: - list_item_num = int(tag.attrs["start"]) - except ValueError: - pass - - for li in tag.find_all("li", recursive=False): - method = getattr(self, "_li", self.inline_tag_to_text) - markup = method(li) - - # li value= attribute will change the item number - # it also overrides any ol start= attribute - - if li.has_attr("value") and len(li.attrs["value"]) > 0: - try: - list_item_num = int(li.attrs["value"]) - except ValueError: - pass - - if not isinstance(markup, urwid.Widget): - txt = self.text_to_widget("li", [str(list_item_num), ". ", markup]) - # 1. foo, 2. bar, etc. - widgets.append(txt) - else: - txt = self.text_to_widget("li", [str(list_item_num), ". "]) - columns = urwid.Columns( - [txt, ("weight", 9999, markup)], dividechars=1, min_width=3 - ) - widgets.append(columns) - - list_item_num += increment - - return urwid.Pile(widgets) - - def _pre(self, tag) -> urwid.Widget: - # tag spec says that text should not wrap,
- # but horizontal screen space is at a premium
- # and we have no horizontal scroll bar, so allow
- # wrapping.
-
- widget_list = [urwid.Divider(" ")]
- widget_list += self.process_block_tag_children(tag)
-
- pre_widget = urwid.Padding(
- urwid.Pile(widget_list),
- align="left",
- width=("relative", 100),
- min_width=None,
- left=1,
- right=1,
- )
- return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
-
- def _span(self, tag) -> Tuple:
- markups = self.process_inline_tag_children(tag)
-
- if not markups:
- return (tag.name, "")
-
- # span inherits its parent's class definition
- # unless it has a specific class definition
- # of its own
-
- if "class" in tag.attrs:
- # uncomment the following code to hide all HTML marked
- # invisible (generally, the http:// prefix of URLs)
- # could be a user preference, it's only advisable if
- # the terminal supports OCS 8 hyperlinks (and that's not
- # automatically detectable)
-
- # if "invisible" in tag.attrs["class"]:
- # return (tag.name, "")
-
- style_name = self.get_urwid_attr_name(tag)
-
- if style_name != "span":
- # unique class name matches an entry in our palette
- return (style_name, markups)
-
- if tag.parent:
- return (self.get_urwid_attr_name(tag.parent), markups)
- else:
- # fallback
- return ("span", markups)
-
- def _strong(self, tag) -> Tuple:
- # to simplify the number of palette entries
- # translate STRONG to B (bold)
- markups = self.process_inline_tag_children(tag)
- if not markups:
- return ("b", "")
-
- # special case processing for bold and italic
- for parent in tag.parents:
- if parent.name == "i" or parent.name == "em":
- return ("bi", markups)
-
- return ("b", markups)
-
- def _ul(self, tag) -> urwid.Widget:
- """unordered list tag handler"""
-
- widgets = []
-
- for li in tag.find_all("li", recursive=False):
- method = getattr(self, "_li", self.inline_tag_to_text)
- markup = method(li)
-
- if not isinstance(markup, urwid.Widget):
- txt = self.text_to_widget("li", ["\N{bullet} ", markup])
- # * foo, * bar, etc.
- widgets.append(txt)
- else:
- txt = self.text_to_widget("li", ["\N{bullet} "])
- columns = urwid.Columns(
- [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
- )
- widgets.append(columns)
-
- return urwid.Pile(widgets)
-
- # These tags are handled identically to others
- # the only difference being the tag name used for
- # urwid attribute mapping
-
- _b = _strong
-
- _div = basic_block_tag_handler
-
- _i = _em
-
- _li = basic_block_tag_handler
-
- # Glitch-soc and Pleroma allow ... in content
- # Mastodon (PR #23913) does not; header tags are converted to
-
- _h1 = _h2 = _h3 = _h4 = _h5 = _h6 = basic_block_tag_handler
-
- _p = basic_block_tag_handler
-
-
-def flatten(data):
- if isinstance(data, tuple):
- for x in data:
- yield from flatten(x)
- else:
- yield data
diff --git a/toot/tui/richtext/__init__.py b/toot/tui/richtext/__init__.py
new file mode 100644
index 0000000..07e31c8
--- /dev/null
+++ b/toot/tui/richtext/__init__.py
@@ -0,0 +1,18 @@
+import urwid
+
+from toot.tui.utils import highlight_hashtags
+from toot.utils import format_content
+from typing import List
+
+try:
+ from .richtext import html_to_widgets, url_to_widget
+except ImportError:
+ # Fallback if urwidgets are not available
+ def html_to_widgets(html: str) -> List[urwid.Widget]:
+ return [
+ urwid.Text(highlight_hashtags(line))
+ for line in format_content(html)
+ ]
+
+ def url_to_widget(url: str):
+ return urwid.Text(("link", url))
diff --git a/toot/tui/richtext/richtext.py b/toot/tui/richtext/richtext.py
new file mode 100644
index 0000000..71897c4
--- /dev/null
+++ b/toot/tui/richtext/richtext.py
@@ -0,0 +1,452 @@
+import re
+import urwid
+import unicodedata
+
+from bs4.element import NavigableString, Tag
+from toot.tui.constants import PALETTE
+from toot.utils import parse_html, urlencode_url
+from typing import List, Tuple
+from urwid.util import decompose_tagmarkup
+from urwidgets import Hyperlink, TextEmbed
+
+
+STYLE_NAMES = [p[0] for p in PALETTE]
+
+# NOTE: update this list if Mastodon starts supporting more block tags
+BLOCK_TAGS = ["p", "pre", "li", "blockquote", "h1", "h2", "h3", "h4", "h5", "h6"]
+
+
+def html_to_widgets(html, recovery_attempt=False) -> List[urwid.Widget]:
+ """Convert html to urwid widgets"""
+ widgets: List[urwid.Widget] = []
+ html = unicodedata.normalize("NFKC", html)
+ soup = parse_html(html)
+
+ first_tag = True
+ for e in soup.body or soup:
+ if isinstance(e, NavigableString):
+ if first_tag and not recovery_attempt:
+ # if our first "tag" is a navigable string
+ # the HTML is out of spec, doesn't start with a tag,
+ # we see this in content from Pixelfed servers.
+ # attempt a fix by wrapping the HTML with
+ return html_to_widgets(f"{html}
", recovery_attempt=True)
+ else:
+ continue
+ else:
+ name = e.name
+ # if our HTML starts with a tag, but not a block tag
+ # the HTML is out of spec. Attempt a fix by wrapping the
+ # HTML with
+ if (first_tag and not recovery_attempt and name not in BLOCK_TAGS):
+ return html_to_widgets(f"{html}
", recovery_attempt=True)
+
+ markup = render(name, e)
+ first_tag = False
+
+ if not isinstance(markup, urwid.Widget):
+ # plaintext, so create a padded text widget
+ txt = text_to_widget("", markup)
+ markup = urwid.Padding(
+ txt,
+ align="left",
+ width=("relative", 100),
+ min_width=None,
+ )
+ widgets.append(markup)
+ # separate top level widgets with a blank line
+ widgets.append(urwid.Divider(" "))
+ return widgets[:-1] # but suppress the last blank line
+
+
+def url_to_widget(url: str):
+ widget = len(url), urwid.Filler(Hyperlink(url, "link", url))
+ return TextEmbed(widget)
+
+
+def inline_tag_to_text(tag) -> Tuple:
+ """Convert html tag to plain text with tag as attributes recursively"""
+ markups = process_inline_tag_children(tag)
+ if not markups:
+ return (tag.name, "")
+ return (tag.name, markups)
+
+
+def process_inline_tag_children(tag) -> List:
+ """Recursively retrieve all children
+ and convert to a list of markup text"""
+ markups = []
+ for child in tag.children:
+ if isinstance(child, Tag):
+ markup = render(child.name, child)
+ markups.append(markup)
+ else:
+ markups.append(child)
+ return markups
+
+
+URL_PATTERN = re.compile(r"(^.+)\x03(.+$)")
+
+
+def text_to_widget(attr, markup) -> urwid.Widget:
+ markup_list = []
+ for run in markup:
+ if isinstance(run, tuple):
+ txt, attr_list = decompose_tagmarkup(run)
+ # find anchor titles with an ETX separator followed by href
+ match = URL_PATTERN.match(txt)
+ if match:
+ label, url = match.groups()
+ anchor_attr = get_best_anchor_attr(attr_list)
+ markup_list.append((
+ len(label),
+ urwid.Filler(Hyperlink(url, anchor_attr, label)),
+ ))
+ else:
+ markup_list.append(run)
+ else:
+ markup_list.append(run)
+
+ return TextEmbed(markup_list)
+
+
+def process_block_tag_children(tag) -> List[urwid.Widget]:
+ """Recursively retrieve all children
+ and convert to a list of widgets
+ any inline tags containing text will be
+ converted to Text widgets"""
+
+ pre_widget_markups = []
+ post_widget_markups = []
+ child_widgets = []
+ found_nested_widget = False
+
+ for child in tag.children:
+ if isinstance(child, Tag):
+ # child is a nested tag; process using custom method
+ # or default to inline_tag_to_text
+ result = render(child.name, child)
+ if isinstance(result, urwid.Widget):
+ found_nested_widget = True
+ child_widgets.append(result)
+ else:
+ if not found_nested_widget:
+ pre_widget_markups.append(result)
+ else:
+ post_widget_markups.append(result)
+ else:
+ # child is text; append to the appropriate markup list
+ if not found_nested_widget:
+ pre_widget_markups.append(child)
+ else:
+ post_widget_markups.append(child)
+
+ widget_list = []
+ if len(pre_widget_markups):
+ widget_list.append(text_to_widget(tag.name, pre_widget_markups))
+
+ if len(child_widgets):
+ widget_list += child_widgets
+
+ if len(post_widget_markups):
+ widget_list.append(text_to_widget(tag.name, post_widget_markups))
+
+ return widget_list
+
+
+def get_urwid_attr_name(tag) -> str:
+ """Get the class name and translate to a
+ name suitable for use as an urwid
+ text attribute name"""
+
+ if "class" in tag.attrs:
+ clss = tag.attrs["class"]
+ if len(clss) > 0:
+ style_name = "class_" + "_".join(clss)
+ # return the class name, only if we
+ # find it as a defined palette name
+ if style_name in STYLE_NAMES:
+ return style_name
+
+ # fallback to returning the tag name
+ return tag.name
+
+
+def basic_block_tag_handler(tag) -> urwid.Widget:
+ """default for block tags that need no special treatment"""
+ return urwid.Pile(process_block_tag_children(tag))
+
+
+def get_best_anchor_attr(attrib_list) -> str:
+ if not attrib_list:
+ return ""
+ flat_al = list(flatten(attrib_list))
+
+ for a in flat_al[0]:
+ # ref: https://docs.joinmastodon.org/spec/activitypub/
+ # these are the class names (translated to attrib names)
+ # that we can support for display
+
+ try:
+ if a[0] in ["class_hashtag", "class_mention_hashtag", "class_mention"]:
+ return a[0]
+ except KeyError:
+ continue
+
+ return "a"
+
+
+def render(attr: str, content: str):
+ if attr in ["a"]:
+ return render_anchor(content)
+
+ if attr in ["blockquote"]:
+ return render_blockquote(content)
+
+ if attr in ["br"]:
+ return render_br(content)
+
+ if attr in ["em"]:
+ return render_em(content)
+
+ if attr in ["ol"]:
+ return render_ol(content)
+
+ if attr in ["pre"]:
+ return render_pre(content)
+
+ if attr in ["span"]:
+ return render_span(content)
+
+ if attr in ["b", "strong"]:
+ return render_strong(content)
+
+ if attr in ["ul"]:
+ return render_ul(content)
+
+ # Glitch-soc and Pleroma allow ... in content
+ # Mastodon (PR #23913) does not; header tags are converted to
+ if attr in ["p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6"]:
+ return basic_block_tag_handler(content)
+
+ # Fall back to inline_tag_to_text handler
+ return inline_tag_to_text(content)
+
+
+def render_anchor(tag) -> Tuple:
+ """anchor tag handler"""
+
+ markups = process_inline_tag_children(tag)
+ if not markups:
+ return (tag.name, "")
+
+ href = tag.attrs["href"]
+ title, attrib_list = decompose_tagmarkup(markups)
+ if not attrib_list:
+ attrib_list = [tag]
+ if href:
+ # urlencode the path and query portions of the URL
+ href = urlencode_url(href)
+ # use ASCII ETX (end of record) as a
+ # delimiter between the title and the HREF
+ title += f"\x03{href}"
+
+ attr = get_best_anchor_attr(attrib_list)
+
+ if attr == "a":
+ # didn't find an attribute to use
+ # in the child markup, so let's
+ # try the anchor tag's own attributes
+
+ attr = get_urwid_attr_name(tag)
+
+ # hashtag anchors have a class of "mention hashtag"
+ # or "hashtag"
+ # we'll return style "class_mention_hashtag"
+ # or "class_hashtag"
+ # in that case; see corresponding palette entry
+ # in constants.py controlling hashtag highlighting
+
+ return (attr, title)
+
+
+def render_blockquote(tag) -> urwid.Widget:
+ widget_list = process_block_tag_children(tag)
+ blockquote_widget = urwid.LineBox(
+ urwid.Padding(
+ urwid.Pile(widget_list),
+ align="left",
+ width=("relative", 100),
+ min_width=None,
+ left=1,
+ right=1,
+ ),
+ tlcorner="",
+ tline="",
+ lline="│",
+ trcorner="",
+ blcorner="",
+ rline="",
+ bline="",
+ brcorner="",
+ )
+ return urwid.Pile([urwid.AttrMap(blockquote_widget, "blockquote")])
+
+
+def render_br(tag) -> Tuple:
+ return ("br", "\n")
+
+
+def render_em(tag) -> Tuple:
+ # to simplify the number of palette entries
+ # translate EM to I (italic)
+ markups = process_inline_tag_children(tag)
+ if not markups:
+ return ("i", "")
+
+ # special case processing for bold and italic
+ for parent in tag.parents:
+ if parent.name == "b" or parent.name == "strong":
+ return ("bi", markups)
+
+ return ("i", markups)
+
+
+def render_ol(tag) -> urwid.Widget:
+ """ordered list tag handler"""
+
+ widgets = []
+ list_item_num = 1
+ increment = -1 if tag.has_attr("reversed") else 1
+
+ # get ol start= attribute if present
+ if tag.has_attr("start") and len(tag.attrs["start"]) > 0:
+ try:
+ list_item_num = int(tag.attrs["start"])
+ except ValueError:
+ pass
+
+ for li in tag.find_all("li", recursive=False):
+ markup = render("li", li)
+
+ # li value= attribute will change the item number
+ # it also overrides any ol start= attribute
+
+ if li.has_attr("value") and len(li.attrs["value"]) > 0:
+ try:
+ list_item_num = int(li.attrs["value"])
+ except ValueError:
+ pass
+
+ if not isinstance(markup, urwid.Widget):
+ txt = text_to_widget("li", [str(list_item_num), ". ", markup])
+ # 1. foo, 2. bar, etc.
+ widgets.append(txt)
+ else:
+ txt = text_to_widget("li", [str(list_item_num), ". "])
+ columns = urwid.Columns(
+ [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+ )
+ widgets.append(columns)
+
+ list_item_num += increment
+
+ return urwid.Pile(widgets)
+
+
+def render_pre(tag) -> urwid.Widget:
+ # tag spec says that text should not wrap,
+ # but horizontal screen space is at a premium
+ # and we have no horizontal scroll bar, so allow
+ # wrapping.
+
+ widget_list = [urwid.Divider(" ")]
+ widget_list += process_block_tag_children(tag)
+
+ pre_widget = urwid.Padding(
+ urwid.Pile(widget_list),
+ align="left",
+ width=("relative", 100),
+ min_width=None,
+ left=1,
+ right=1,
+ )
+ return urwid.Pile([urwid.AttrMap(pre_widget, "pre")])
+
+
+def render_span(tag) -> Tuple:
+ markups = process_inline_tag_children(tag)
+
+ if not markups:
+ return (tag.name, "")
+
+ # span inherits its parent's class definition
+ # unless it has a specific class definition
+ # of its own
+
+ if "class" in tag.attrs:
+ # uncomment the following code to hide all HTML marked
+ # invisible (generally, the http:// prefix of URLs)
+ # could be a user preference, it's only advisable if
+ # the terminal supports OCS 8 hyperlinks (and that's not
+ # automatically detectable)
+
+ # if "invisible" in tag.attrs["class"]:
+ # return (tag.name, "")
+
+ style_name = get_urwid_attr_name(tag)
+
+ if style_name != "span":
+ # unique class name matches an entry in our palette
+ return (style_name, markups)
+
+ if tag.parent:
+ return (get_urwid_attr_name(tag.parent), markups)
+ else:
+ # fallback
+ return ("span", markups)
+
+
+def render_strong(tag) -> Tuple:
+ # to simplify the number of palette entries
+ # translate STRONG to B (bold)
+ markups = process_inline_tag_children(tag)
+ if not markups:
+ return ("b", "")
+
+ # special case processing for bold and italic
+ for parent in tag.parents:
+ if parent.name == "i" or parent.name == "em":
+ return ("bi", markups)
+
+ return ("b", markups)
+
+
+def render_ul(tag) -> urwid.Widget:
+ """unordered list tag handler"""
+
+ widgets = []
+
+ for li in tag.find_all("li", recursive=False):
+ markup = render("li", li)
+
+ if not isinstance(markup, urwid.Widget):
+ txt = text_to_widget("li", ["\N{bullet} ", markup])
+ # * foo, * bar, etc.
+ widgets.append(txt)
+ else:
+ txt = text_to_widget("li", ["\N{bullet} "])
+ columns = urwid.Columns(
+ [txt, ("weight", 9999, markup)], dividechars=1, min_width=3
+ )
+ widgets.append(columns)
+
+ return urwid.Pile(widgets)
+
+
+def flatten(data):
+ if isinstance(data, tuple):
+ for x in data:
+ yield from flatten(x)
+ else:
+ yield data
diff --git a/toot/tui/stubs/stub_hyperlink.py b/toot/tui/stubs/stub_hyperlink.py
deleted file mode 100644
index a2831ba..0000000
--- a/toot/tui/stubs/stub_hyperlink.py
+++ /dev/null
@@ -1,30 +0,0 @@
-__all__ = ("Hyperlink",)
-
-import urwid
-
-
-class Hyperlink(urwid.WidgetWrap):
- def __init__(
- self,
- uri,
- attr,
- text,
- ):
- pass
-
- def render(self, size, focus):
- return None
-
-
-class HyperlinkCanvas(urwid.Canvas):
- def __init__(self, uri: str, text_canv: urwid.TextCanvas):
- pass
-
- def cols(self):
- return 0
-
- def content(self, *args, **kwargs):
- yield [None]
-
- def rows(self):
- return 0
diff --git a/toot/tui/stubs/stub_text_embed.py b/toot/tui/stubs/stub_text_embed.py
deleted file mode 100644
index bf587d2..0000000
--- a/toot/tui/stubs/stub_text_embed.py
+++ /dev/null
@@ -1,29 +0,0 @@
-__all__ = ("parse_text", "TextEmbed")
-
-import urwid
-
-
-class TextEmbed(urwid.Text):
- def get_text(
- self,
- ):
- return None
-
- def render(self, size, focus):
- return None
-
- def set_text(self, markup):
- pass
-
- def set_wrap_mode(self, mode):
- pass
-
-
-def parse_text(
- text,
- patterns,
- repl,
- *repl_args,
- **repl_kwargs,
-):
- return None
diff --git a/toot/tui/stubs/urwidgets.py b/toot/tui/stubs/urwidgets.py
deleted file mode 100644
index 92737d3..0000000
--- a/toot/tui/stubs/urwidgets.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# If urwidgets is loaded use it; otherwise use our stubs
-try:
- from urwidgets import Hyperlink, TextEmbed, parse_text
- has_urwidgets = True
-except ImportError:
- from .stub_hyperlink import Hyperlink
- from .stub_text_embed import TextEmbed, parse_text
- has_urwidgets = False
diff --git a/toot/tui/timeline.py b/toot/tui/timeline.py
index 1fef40c..93421ce 100644
--- a/toot/tui/timeline.py
+++ b/toot/tui/timeline.py
@@ -1,11 +1,11 @@
import logging
-import re
import urwid
import webbrowser
from typing import List, Optional
from toot.tui import app
+from toot.tui.richtext import html_to_widgets, url_to_widget
from toot.utils.datetime import parse_datetime, time_ago
from toot.utils.language import language_name
@@ -13,9 +13,6 @@ from toot.entities import Status
from toot.tui.scroll import Scrollable, ScrollBar
from toot.tui.utils import highlight_keys
from toot.tui.widgets import SelectableText, SelectableColumns
-from toot.tui.richtext import ContentParser
-from toot.utils import urlencode_url
-from toot.tui.stubs.urwidgets import Hyperlink, TextEmbed, parse_text, has_urwidgets
logger = logging.getLogger("toot")
@@ -320,20 +317,6 @@ class StatusDetails(urwid.Pile):
if status else ())
return super().__init__(widget_list)
- def linkify_content(self, text) -> urwid.Widget:
- if not has_urwidgets:
- return urwid.Text(("link", text))
- TRANSFORM = {
- # convert http[s] URLs to Hyperlink widgets for nesting in a TextEmbed widget
- re.compile(r'(https?://[^\s]+)'):
- lambda g: (len(g[1]), urwid.Filler(Hyperlink(urlencode_url(g[1]), "link", g[1]))),
- }
- markup_list = []
-
- markup_list.append(parse_text(text, TRANSFORM,
- lambda pattern, groups, span: TRANSFORM[pattern](groups)))
- return TextEmbed(markup_list, align='left')
-
def content_generator(self, status, reblogged_by):
if reblogged_by:
text = "♺ {} boosted".format(reblogged_by.display_name or reblogged_by.username)
@@ -356,9 +339,7 @@ class StatusDetails(urwid.Pile):
yield ("pack", urwid.Text(("content_warning", "Marked as sensitive. Press S to view.")))
else:
content = status.original.translation if status.original.show_translation else status.data["content"]
-
- parser = ContentParser()
- widgetlist = parser.html_to_widgets(content)
+ widgetlist = html_to_widgets(content)
for line in widgetlist:
yield (line)
@@ -370,7 +351,7 @@ class StatusDetails(urwid.Pile):
yield ("pack", urwid.Text([("bold", "Media attachment"), " (", m["type"], ")"]))
if m["description"]:
yield ("pack", urwid.Text(m["description"]))
- yield ("pack", self.linkify_content(m["url"]))
+ yield ("pack", url_to_widget(m["url"]))
poll = status.original.data.get("poll")
if poll:
@@ -430,7 +411,7 @@ class StatusDetails(urwid.Pile):
if card["description"]:
yield urwid.Text(card["description"].strip())
yield urwid.Text("")
- yield self.linkify_content(card["url"])
+ yield url_to_widget(card["url"])
def poll_generator(self, poll):
for idx, option in enumerate(poll["options"]):
diff --git a/toot/tui/urwidgets.py b/toot/tui/urwidgets.py
deleted file mode 100644
index ee731a8..0000000
--- a/toot/tui/urwidgets.py
+++ /dev/null
@@ -1,8 +0,0 @@
-# If urwidgets is loaded use it; otherwise use our stubs
-try:
- from urwidgets import Hyperlink, TextEmbed, parse_text # noqa: F401
- has_urwidgets = True
-except ImportError:
- from .stub_hyperlink import Hyperlink # noqa: F401
- from .stub_text_embed import TextEmbed, parse_text # noqa: F401
- has_urwidgets = False
diff --git a/toot/tui/utils.py b/toot/tui/utils.py
index 0ccff9d..734ae32 100644
--- a/toot/tui/utils.py
+++ b/toot/tui/utils.py
@@ -35,6 +35,18 @@ def highlight_keys(text, high_attr, low_attr=""):
return list(_gen())
+def highlight_hashtags(line):
+ hline = []
+
+ for p in re.split(HASHTAG_PATTERN, line):
+ if p.startswith("#"):
+ hline.append(("hashtag", p))
+ else:
+ hline.append(p)
+
+ return hline
+
+
def show_media(paths):
"""
Attempt to open an image viewer to show given media files.
diff --git a/toot/utils/__init__.py b/toot/utils/__init__.py
index 43af373..c4afa7f 100644
--- a/toot/utils/__init__.py
+++ b/toot/utils/__init__.py
@@ -23,20 +23,22 @@ def str_bool_nullable(b):
return None if b is None else str_bool(b)
-def get_text(html):
- """Converts html to text, strips all tags."""
-
+def parse_html(html: str) -> BeautifulSoup:
# Ignore warnings made by BeautifulSoup, if passed something that looks like
# a file (e.g. a dot which matches current dict), it will warn that the file
# should be opened instead of passing a filename.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
- text = BeautifulSoup(html.replace(''', "'"), "html.parser").get_text()
-
- return unicodedata.normalize('NFKC', text)
+ return BeautifulSoup(html.replace("'", "'"), "html.parser")
-def parse_html(html):
+def get_text(html):
+ """Converts html to text, strips all tags."""
+ text = parse_html(html).get_text()
+ return unicodedata.normalize("NFKC", text)
+
+
+def html_to_paragraphs(html):
"""Attempt to convert html to plain text while keeping line breaks.
Returns a list of paragraphs, each being a list of lines.
"""
@@ -55,7 +57,7 @@ def format_content(content):
Returns a generator yielding lines of content.
"""
- paragraphs = parse_html(content)
+ paragraphs = html_to_paragraphs(content)
first = True