Files
MDHelper/mdhelper/batch_documents.py
T

151 lines
7.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Immutable multi-document composition and transactional publication."""
from __future__ import annotations
import hashlib
import html
import re
import shutil
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import unquote, urlsplit, urlunsplit
from .document_resources import analyze_document, apply_source_edits, resolve_local_reference, SourceEdit
from .doc_check import document_anchors
from .export_jobs import check_cancelled
from .file_versions import atomic_write
from .markdown_utils import Heading
@dataclass(frozen=True)
class BatchDocument:
path: Path
source: str
@dataclass
class CombinedDocument:
word_source: str
html_body: str
headings: list[Heading]
def combine_documents(documents: list[BatchDocument], renderer, cancel=None) -> CombinedDocument:
documents = [BatchDocument(d.path.resolve(), d.source) for d in documents]
mappings, analyses = {}, {}
for index, document in enumerate(documents, 1):
check_cancelled(cancel)
analysis = analyses[document.path] = analyze_document(document.source)
mapping = {"": f"mdh{index}"}
for number, token in enumerate(t for t in analysis.tokens if t.type == "heading_open"):
if token.attrGet("id"):
mapping[token.attrGet("id")] = f"mdh{index}h{number + 1}"
for number, anchor in enumerate(sorted(document_anchors(analysis) - mapping.keys()), 1):
mapping[anchor] = f"mdh{index}a{number}"
mappings[document.path] = mapping
word, bodies, headings = [], [], []
for document in documents:
check_cancelled(cancel)
mapping = mappings[document.path]
edits, word_resource_edits = [], []
for reference in analyses[document.path].references:
parsed = urlsplit(reference.url)
target = resolve_local_reference(document.path, reference.url)
local = parsed.scheme in {"", "file"} and (not parsed.netloc or parsed.scheme == "file")
if local and not parsed.path:
target = document.path
new_url = reference.url
if not reference.image and target in mappings and unquote(parsed.fragment) in mappings[target]:
new_url = "#" + mappings[target][unquote(parsed.fragment)]
elif target is not None and parsed.path:
new_url = urlunsplit(urlsplit(target.as_uri())._replace(query=parsed.query, fragment=parsed.fragment))
# Expand reference-style links before joining documents with identical definitions.
if new_url != reference.url or reference.syntax == "reference":
edits.append(reference.edit_url(new_url))
word_url = new_url
if reference.image and target is not None:
# Pandoc does not decode file-URI escapes on Windows. Angle-bracket
# destinations preserve literal Unicode, spaces and parentheses.
word_url = target.as_posix()
if reference.syntax != 'html' and document.source[reference.start - 1:reference.start] != '<':
word_url = '<' + word_url + '>'
word_resource_edits.append(reference.edit_url(word_url))
source = apply_source_edits(document.source, edits)
word_source = apply_source_edits(document.source, word_resource_edits)
env = {"path": str(document.path), "headings": []}
tokens = renderer.markdown.parse(source, env)
offsets = [0]
for line in word_source.splitlines(keepends=True): offsets.append(offsets[-1] + len(line))
word_tokens = renderer.markdown.parse(word_source, {})
word_heading_texts = iter(word_tokens[i + 1].content for i, token in enumerate(word_tokens) if token.type == 'heading_open')
word_edits = []
heading_number = 0
for index, token in enumerate(tokens):
if token.type == "heading_open":
heading_number += 1
anchor = f"{mapping['']}h{heading_number}"
token.attrSet("id", anchor)
level = min(6, int(token.tag[1:]) + 1)
token.tag = f"h{level}"
content = tokens[index + 1].content
headings.append(Heading(level, content, anchor))
start, end = token.map
prefix = re.match(r"[ \t]*(?:>[ \t]*)*", word_source[offsets[start]:]).group()
word_edits.append(SourceEdit(offsets[start], offsets[end], f"{prefix}{'#' * level} {next(word_heading_texts)} {{#{anchor}}}\n"))
elif token.type == "heading_close":
token.tag = f"h{min(6, int(token.tag[1:]) + 1)}"
for raw in [token, *(token.children or [])]:
if raw.type in {"html_inline", "html_block"}:
raw.content = re.sub(r'\b(id|name)\s*=\s*([\"\x27])([^\"\x27]+)\2',
lambda m: f'{m[1]}={m[2]}{mapping.get(html.unescape(m[3]), m[3])}{m[2]}', raw.content)
title = document.path.stem
chapter = Heading(1, title, mapping[""])
first = len(headings) - sum(1 for token in tokens if token.type == "heading_open")
headings.insert(first, chapter)
safe_title = re.sub(r'([\\`*_{}\[\]<>#])', r'\\\1', title)
word.append(f"# {safe_title} {{#{mapping['']}}}\n\n" + apply_source_edits(word_source, word_edits))
bodies.append(f'<section class="batch-chapter"><h1 id="{mapping[""]}">{html.escape(title)}</h1>' +
renderer.markdown.renderer.render(tokens, renderer.markdown.options, env) + '</section>')
return CombinedDocument("\n\n".join(word), "\n".join(bodies), headings)
def fingerprint(path: Path):
if not path.exists(): return None
digest = hashlib.sha256()
with path.open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''): digest.update(block)
return digest.hexdigest()
def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, cancel=None):
"""Publish only after all conversions succeed; restore earlier outputs on any failure."""
check_cancelled(cancel)
originals = {}
for index, destination in enumerate(outputs):
check_cancelled(cancel)
if fingerprint(destination) != expected[destination]:
raise OSError(f"导出期间目标文件发生变化,已停止替换:{destination}")
if destination.exists():
copy = backup / f"previous-{index}"
shutil.copyfile(destination, copy)
originals[destination] = copy
else:
originals[destination] = None
written = []
try:
for destination, staged in outputs.items():
check_cancelled(cancel)
atomic_write(destination, staged.read_bytes())
written.append(destination)
check_cancelled(cancel)
except Exception as error:
failures = []
for destination in reversed(written):
try:
if originals[destination] is None: destination.unlink(missing_ok=True)
else: atomic_write(destination, originals[destination].read_bytes())
except OSError as rollback:
failures.append(str(rollback))
if failures:
raise OSError(f"输出恢复失败,备份位于 {backup}:{';'.join(failures)}") from error
raise
return list(outputs)