"""Shared Markdown/HTML resource discovery with source spans for safe editing."""
from __future__ import annotations
import bisect
import re
from dataclasses import dataclass
from html import escape, unescape
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import unquote, urlsplit
from urllib.request import url2pathname
@dataclass(frozen=True)
class SourceEdit:
start: int
end: int
text: str
@dataclass(frozen=True)
class ResourceReference:
url: str
line: int
image: bool
start: int
end: int
syntax: str = "markdown"
label: str = ""
title: str = ""
def edit_url(self, url: str) -> SourceEdit:
if self.syntax == "html":
text = '"' + escape(url, quote=True) + '"'
elif self.syntax == "reference":
title = ' "' + self.title.replace("\\", "\\\\").replace('"', '\\"') + '"' if self.title else ""
text = self.label + "(" + url + title + ")"
else:
text = url
return SourceEdit(self.start, self.end, text)
@dataclass
class DocumentAnalysis:
tokens: list
references: list[ResourceReference]
def resolve_local_reference(document: Path, url: str) -> Path | None:
parsed = urlsplit(url)
if not parsed.path or parsed.scheme not in {"", "file"}:
return None
if parsed.scheme == "file":
return Path(url2pathname(("//" + parsed.netloc if parsed.netloc else "") + parsed.path)).resolve()
if parsed.netloc:
return None
return (document.parent / unquote(parsed.path)).resolve()
def apply_source_edits(source: str, edits: list[SourceEdit]) -> str:
pieces, offset = [], 0
for edit in sorted(edits, key=lambda item: item.start):
if edit.start < offset:
raise ValueError("Overlapping source edits")
pieces.extend((source[offset:edit.start], edit.text))
offset = edit.end
return "".join([*pieces, source[offset:]])
class _HtmlReferences(HTMLParser):
def __init__(self, raw, offset, line_number, references):
super().__init__(convert_charrefs=False)
self.document_offset, self.line_number, self.references = offset, line_number, references
self.lines = [0] + [m.end() for m in re.finditer("\n", raw)]
self.feed(raw)
def handle_starttag(self, tag, attrs):
raw = self.get_starttag_text()
tag_end = re.match(r"<\s*[^\s/>]+", raw).end()
pattern = r'''([^\s=<>/]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]+))?'''
line, column = self.getpos()
offset = self.document_offset + self.lines[line - 1] + column + tag_end
for match in re.finditer(pattern, raw[tag_end:]):
name, value = match[1].lower(), match[2]
if name not in {"src", "href"} or value is None:
continue
quoted = value[:1] in {"'", '"'}
url = unescape(value[1:-1] if quoted else value)
self.references.append(ResourceReference(url, self.line_number + line - 1, tag == "img" and name == "src",
offset + match.start(2), offset + match.end(2), "html"))
class _ResourceScanner:
def __init__(self, source: str):
from .markdown_renderer import MarkdownRenderer
self.source = source
self.parser = MarkdownRenderer(Path.cwd()).markdown
self.env = {}
self.tokens = self.parser.parse(source, self.env)
self.offsets = [0]
for line in source.splitlines(keepends=True):
self.offsets.append(self.offsets[-1] + len(line))
self.references: list[ResourceReference] = []
def line_at(self, offset: int) -> int:
return bisect.bisect_right(self.offsets, offset)
def masked_source(self) -> str:
masked, excluded = list(self.source), []
for token in self.tokens:
if token.map and token.type in {"fence", "code_block", "math_block", "html_block"}:
start, end = (self.offsets[n] for n in token.map)
if token.type == "html_block":
_HtmlReferences(self.source[start:end], start, self.line_at(start), self.references)
excluded.append((start, end))
for ref in self.env.get("references", {}).values():
excluded.append(tuple(self.offsets[n] for n in ref["map"]))
for start, end in excluded:
masked[start:end] = ["\n" if c == "\n" else " " for c in self.source[start:end]]
return "".join(masked)
def capture_destination(self, state, start, token, image):
label_end = self.parser.helpers.parseLinkLabel(state, start + int(image), False)
pos = label_end + 1
url, title = token.attrGet("src" if image else "href") or "", token.attrGet("title") or ""
if pos < len(self.source) and self.source[pos] == "(":
pos += 1
while self.source[pos].isspace():
pos += 1
destination = self.parser.helpers.parseLinkDestination(self.source, pos, len(self.source))
reference = ResourceReference(url, self.line_at(start), image, pos, destination.pos)
else:
reference = ResourceReference(url, self.line_at(start), image, start, state.pos, "reference",
self.source[start:label_end + 1], title)
self.references.append(reference)
def scan(self) -> DocumentAnalysis:
from markdown_it.rules_inline import image, link, html_inline
inline_source = self.masked_source()
def capture(rule, is_image):
def parse(state, silent):
start, token_start = state.pos, len(state.tokens)
matched = rule(state, silent)
if matched and not silent and state.src is inline_source:
token = next(t for t in state.tokens[token_start:] if t.type == ("image" if is_image else "link_open"))
self.capture_destination(state, start, token, is_image)
return matched
return parse
def capture_html(state, silent):
start = state.pos
matched = html_inline(state, silent)
if matched and not silent and state.src is inline_source:
_HtmlReferences(self.source[start:state.pos], start, self.line_at(start), self.references)
return matched
self.parser.inline.ruler.at("image", capture(image, True))
self.parser.inline.ruler.at("link", capture(link, False))
self.parser.inline.ruler.at("html_inline", capture_html)
self.parser.inline.parse(inline_source, self.parser, self.env, [])
return DocumentAnalysis(self.tokens, sorted(self.references, key=lambda r: r.start))
def analyze_document(source: str) -> DocumentAnalysis:
return _ResourceScanner(source).scan()