diff --git a/mdhelper/batch_documents.py b/mdhelper/batch_documents.py index 7525cce..b7899c9 100644 --- a/mdhelper/batch_documents.py +++ b/mdhelper/batch_documents.py @@ -11,7 +11,7 @@ from urllib.parse import unquote, urlsplit, urlunsplit from .document_resources import analyze_document, apply_source_edits, resolve_local_reference, SourceEdit from .doc_check import document_anchors from .export_jobs import check_cancelled -from .file_versions import atomic_write +from .file_versions import atomic_write, fingerprint, checked_remove from .markdown_utils import Heading @@ -88,7 +88,7 @@ def combine_documents(documents: list[BatchDocument], renderer, cancel=None) -> content = tokens[index + 1].content headings.append(Heading(level, content, anchor)) start, end = token.map - prefix = re.match(r"[ \t]*(?:>[ \t]*)*", word_source[offsets[start]:]).group() + prefix = re.match(r"[ \t]*(?:>[ \t]*|(?:[-+*]|\d+[.)])[ \t]+)*", word_source[offsets[start]:]).group() word_edits.append(SourceEdit(offsets[start], offsets[end], f"{prefix}{'#' * level} {next(word_heading_texts)} {{#{anchor}}}\n")) elif token.type == "heading_close": token.tag = f"h{min(6, int(token.tag[1:]) + 1)}" @@ -107,14 +107,6 @@ def combine_documents(documents: list[BatchDocument], renderer, cancel=None) -> return CombinedDocument("\n\n".join(word), "\n".join(bodies), headings) -def fingerprint(path: Path): - if not path.exists(): return None - digest = hashlib.sha256() - with path.open('rb') as stream: - for block in iter(lambda: stream.read(1024 * 1024), b''): digest.update(block) - return digest.hexdigest() - - def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, cancel=None): """Publish only after all conversions succeed; restore earlier outputs on any failure.""" check_cancelled(cancel) @@ -140,7 +132,7 @@ def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, can # destination again immediately before publishing this particular file. if fingerprint(destination) != expected[destination]: raise OSError(f"导出期间目标文件发生变化,已停止替换:{destination}") - atomic_write(destination, data) + atomic_write(destination, data, expected=expected[destination]) written.append((destination, hashlib.sha256(data).hexdigest())) check_cancelled(cancel) except Exception as error: @@ -149,8 +141,8 @@ def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, can try: if fingerprint(destination) != published: raise OSError(f"目标文件已被外部修改,保留当前内容:{destination}") - if originals[destination] is None: destination.unlink(missing_ok=True) - else: atomic_write(destination, originals[destination].read_bytes()) + if originals[destination] is None: checked_remove(destination, published) + else: atomic_write(destination, originals[destination].read_bytes(), expected=published) except OSError as rollback: failures.append(str(rollback)) if failures: diff --git a/mdhelper/docx_exporter.py b/mdhelper/docx_exporter.py index adf114f..c19aa29 100644 --- a/mdhelper/docx_exporter.py +++ b/mdhelper/docx_exporter.py @@ -141,7 +141,7 @@ def export_markdown_to_docx_pandoc(source: str, output_path: Path, source_path: # Export the current editor text, including unsaved edits. Retain the source # directory only for resolving linked images and other local resources. with tempfile.NamedTemporaryFile("w", suffix=".md", encoding="utf-8", delete=False) as handle: - handle.write(source) + handle.write(_pandoc_list_tables(source)) temporary_path = Path(handle.name) input_path = temporary_path resource_directory = source_path.resolve().parent if source_path is not None else input_path.parent @@ -233,6 +233,24 @@ def _markdown_blocks(source: str) -> list[ParagraphBlock | TableBlock]: return _DocxBuilder(markdown.parse(source)).build() +def _pandoc_list_tables(source: str) -> str: + """Disambiguate table-first list items for Pandoc's Markdown reader.""" + markdown = MarkdownIt('commonmark').enable('table') + lines = source.splitlines(keepends=True) + items = [] + for token in markdown.parse(source): + if token.type == 'list_item_open': items.append(token.map[0]) + elif token.type == 'list_item_close': items.pop() + elif token.type == 'table_open' and items and token.map[0] == items[-1]: + row = token.map[0] + match = re.match(r'([ \t]*(?:>[ \t]*)*)([-+*]|\d+[.)])([ \t]+)', lines[row]) + if match: + prefix, marker, spacing = match.groups() + indent = prefix + ' ' * (len(marker) + len(spacing)) + lines[row] = match.group() + '\n' + prefix + '\n' + indent + lines[row][match.end():] + return ''.join(lines) + + def _add_python_docx_paragraph(document: object, block: ParagraphBlock) -> None: style = f"Heading {block.style.removeprefix('Heading')}" if block.style.startswith("Heading") else None special = block.style.startswith('Heading') or block.style == 'Code' @@ -398,6 +416,8 @@ class _DocxBuilder: self.list_starts.pop() self.list_quote_depths.pop() elif token.type == "table_open": + if self.item_pending and self.item_pending[-1]: + self._append_paragraph([], 'Normal', self.list_stack[-1]) self.blocks.append(self._collect_table()) continue elif token.type == "hr": diff --git a/mdhelper/docx_semantic.py b/mdhelper/docx_semantic.py index 0f27def..eab66f8 100644 --- a/mdhelper/docx_semantic.py +++ b/mdhelper/docx_semantic.py @@ -509,6 +509,9 @@ class WordStyles: ppr = properties(paragraph, 'pPr') original_id = reference(ppr, 'pStyle') or 'Normal' names = {reference(style, 'name') for style in self.chain(original_id)} + if 'MDHelperSourceTableMarker' in names: + for text in paragraph.iter(W + 't'): + if text.text == '\u200b': text.text = '' for name in tuple(names): if name in SOURCE_LIST_STYLES: base, list_target = SOURCE_LIST_STYLES[name] @@ -584,7 +587,7 @@ def apply_semantic_styles(path, rules, **defaults): if any(source_style_order(reference(s, 'name')) for s in adapter.chain(identifier)): adapter.apply_run(run, 'paragraph') for style in list(parts['word/styles.xml']): - if source_style_order(reference(style, 'name')) or reference(style, 'name') in SOURCE_PARAGRAPH_STYLES: + if source_style_order(reference(style, 'name')) or reference(style, 'name') in (*SOURCE_PARAGRAPH_STYLES, 'MDHelperSourceTableMarker'): parts['word/styles.xml'].remove(style) for filename, root in parts.items(): _order_properties(root) diff --git a/mdhelper/file_versions.py b/mdhelper/file_versions.py index 8fc9a40..b780231 100644 --- a/mdhelper/file_versions.py +++ b/mdhelper/file_versions.py @@ -8,8 +8,90 @@ import time import uuid from pathlib import Path +_UNCHECKED = object() -def atomic_write(path: Path, data: bytes) -> None: + +def fingerprint(path: Path): + try: + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + except FileNotFoundError: + return None + + +def _replace_with_backup(temporary: Path, path: Path, backup: Path) -> None: + """Retain the exact file displaced at the commit boundary, not an earlier copy.""" + if os.name != 'nt': + # Never silently fall back to an unchecked overwrite on another platform. + raise OSError('当前平台不支持受保护的文件替换。') + import ctypes + from ctypes import wintypes + replace = ctypes.WinDLL('kernel32', use_last_error=True).ReplaceFileW + replace.argtypes = [wintypes.LPCWSTR] * 3 + [wintypes.DWORD, wintypes.LPVOID, wintypes.LPVOID] + replace.restype = wintypes.BOOL + if not replace(str(path), str(temporary), str(backup), 0, None, None): + raise ctypes.WinError(ctypes.get_last_error()) + + +def _recovery_path(path: Path) -> Path: + return path.with_name(f'.{path.name}.conflict-{uuid.uuid4().hex}.bak') + + +def _checked_replace(temporary: Path, path: Path, expected: str | None) -> None: + if fingerprint(path) != expected: + raise OSError(f'文件发生变化,已停止替换:{path}') + if expected is None: + # Creating the directory entry must fail if another process got there first. + os.link(temporary, path) + return + backup = _recovery_path(path) + try: + published = fingerprint(temporary) + _replace_with_backup(temporary, path, backup) + if fingerprint(backup) == expected: + try: + backup.unlink() + except OSError: + pass # A retained backup must not turn a committed write into failure. + return + # Retain the conflicting version even if another process saves again + # during recovery. Every displaced file has its own recovery name. + if fingerprint(path) == published: + descriptor, restore_name = tempfile.mkstemp(dir=path.parent, suffix='.tmp') + restore = Path(restore_name) + try: + with os.fdopen(descriptor, 'wb') as stream: + stream.write(backup.read_bytes()) + displaced = _recovery_path(path) + _replace_with_backup(restore, path, displaced) + if fingerprint(displaced) == published: + displaced.unlink() + finally: + restore.unlink(missing_ok=True) + raise OSError(f'文件在替换时发生变化,已保留冲突版本:{backup}') + except OSError as exc: + if backup.exists(): + raise OSError(f'文件替换未完成,恢复副本位于 {backup}:{exc}') from exc + raise + + +def checked_remove(path: Path, expected: str) -> None: + """Rollback a newly created output without deleting a concurrent save.""" + if fingerprint(path) != expected: + raise OSError(f'文件已被外部修改,保留当前内容:{path}') + recovery = _recovery_path(path) + path.rename(recovery) + if fingerprint(recovery) == expected: + recovery.unlink() + else: + try: + os.link(recovery, path) # Exclusive restore; never overwrite a newer file. + except OSError: + pass + raise OSError(f'文件在恢复时发生变化,保留副本:{recovery}') + + +def atomic_write(path: Path, data: bytes, *, expected=_UNCHECKED) -> None: path = path.resolve() path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) @@ -18,7 +100,10 @@ def atomic_write(path: Path, data: bytes) -> None: stream.write(data) stream.flush() os.fsync(stream.fileno()) - os.replace(temporary, path) + if expected is _UNCHECKED: + os.replace(temporary, path) + else: + _checked_replace(Path(temporary), path, expected) finally: Path(temporary).unlink(missing_ok=True) diff --git a/mdhelper/rename_references.py b/mdhelper/rename_references.py index 9067ddf..b8e8d0a 100644 --- a/mdhelper/rename_references.py +++ b/mdhelper/rename_references.py @@ -2,6 +2,7 @@ from __future__ import annotations import difflib +import hashlib import json import os import time @@ -91,7 +92,7 @@ def execute_rename(plan: RenamePlan, backup_root: Path | None = None) -> Path: destination = plan.target if path == plan.source else path if destination.read_bytes() != plan.originals[path]: raise OSError(f"文件在重命名期间已变化,保留当前内容:{destination.name}") - atomic_write(destination, data) + atomic_write(destination, data, expected=hashlib.sha256(plan.originals[path]).hexdigest()) written.append(path) except OSError as exc: failures = [] @@ -100,7 +101,7 @@ def execute_rename(plan: RenamePlan, backup_root: Path | None = None) -> Path: destination = plan.target if path == plan.source else path if destination.read_bytes() != plan.replacements[path]: raise OSError(f"文件已被外部修改,保留当前内容:{destination.name}") - atomic_write(destination, plan.originals[path]) + atomic_write(destination, plan.originals[path], expected=hashlib.sha256(plan.replacements[path]).hexdigest()) except OSError as rollback_error: failures.append(str(rollback_error)) if renamed: diff --git a/mdhelper/resources/word_inline_styles.lua b/mdhelper/resources/word_inline_styles.lua index 6cdc454..5f40e34 100644 --- a/mdhelper/resources/word_inline_styles.lua +++ b/mdhelper/resources/word_inline_styles.lua @@ -48,6 +48,12 @@ local function mark(items, stack) end local function list_paragraphs(list) for _, item in ipairs(list.content) do + -- Word numbers paragraphs, not tables. Give a table-first item its own + -- empty numbered paragraph; its following text remains a continuation. + if item[1] and item[1].t == 'Table' then + item:insert(1, pandoc.Div({pandoc.Para({pandoc.Str('\u{200b}')})}, + pandoc.Attr('', {}, {['custom-style']='MDHelperSourceTableMarker'}))) + end local target = list.t == 'OrderedList' and 'list_ordered' or 'list_bullet' local text = pandoc.utils.stringify(item[1] or {}) if text:match('^%[[ xX]%]') or text:sub(1, #'☐') == '☐' or text:sub(1, #'☒') == '☒' or text:sub(1, #'☑') == '☑' then diff --git a/tests/test_batch_export.py b/tests/test_batch_export.py index b770820..add7290 100644 --- a/tests/test_batch_export.py +++ b/tests/test_batch_export.py @@ -53,9 +53,9 @@ def test_publication_cancellation_race_and_rollback(tmp_path, monkeypatch): with pytest.raises(OSError, match='发生变化'): publish_outputs(outputs, expected, tmp_path) destinations[0].write_bytes(b'old') real_write = batch_documents.atomic_write - def fail(path, data): + def fail(path, data, **kwargs): if path == destinations[1] and data == b'new': raise OSError('disk full') - real_write(path, data) + real_write(path, data, **kwargs) monkeypatch.setattr(batch_documents, 'atomic_write', fail) with pytest.raises(OSError, match='disk full'): publish_outputs(outputs, expected, tmp_path) assert all(path.read_bytes() == b'old' for path in destinations) diff --git a/tests/test_content_revision_057.py b/tests/test_content_revision_057.py new file mode 100644 index 0000000..159d6b9 --- /dev/null +++ b/tests/test_content_revision_057.py @@ -0,0 +1,101 @@ +"""Commit-boundary conflicts and preservation of list content in Word.""" +import zipfile +from lxml import etree as ET +import pytest +from mdhelper import file_versions as files, batch_documents as batch, rename_references as rename +from mdhelper.docx_exporter import export_markdown_to_docx +from mdhelper.markdown_renderer import MarkdownRenderer +from mdhelper.docx_style import W, NS +from test_style_regressions import parts + +TABLE_LIST = '1. | HeadOne | HeadTwo |\n |---|---|\n | CellOne | CellTwo |\n\n ContinuedText\n\n2. LastText' +HEADING_LIST = '1. ### FirstHeading\n\n ContinuedText\n\n2. ### LastHeading' + + +@pytest.mark.parametrize('operation', ['batch', 'rename']) +@pytest.mark.parametrize('boundary', ['flush', 'replace']) +def test_transactions_retain_external_save_at_commit(tmp_path, monkeypatch, operation, boundary): + target = tmp_path/'ref.md' + target.write_bytes(b'[Link](old.md)') + stage = tmp_path/'stage'; stage.write_bytes(b'Exported') + source = tmp_path/'old.md'; source.write_bytes(b'Source') + plan = rename.plan_rename(tmp_path, source, tmp_path/'new.md') + expected = {target: files.fingerprint(target)} + replace, fsync = files._replace_with_backup, files.os.fsync + external = b'External save at final commit boundary' + injected = False + def race_replace(temporary, path, backup): + nonlocal injected + if path == target and not injected: + injected = True; target.write_bytes(external) + replace(temporary, path, backup) + def race_flush(descriptor): + nonlocal injected + fsync(descriptor) + if not injected: + injected = True; target.write_bytes(external) + monkeypatch.setattr(files, '_replace_with_backup', race_replace) if boundary == 'replace' else monkeypatch.setattr(files.os, 'fsync', race_flush) + with pytest.raises(OSError): + if operation == 'batch': batch.publish_outputs({target: stage}, expected, tmp_path) + else: rename.execute_rename(plan, tmp_path/'backups') + assert target.read_bytes() == external + assert source.exists() and not (tmp_path/'new.md').exists() + + +def test_creation_does_not_overwrite_file_created_at_commit(tmp_path, monkeypatch): + path = tmp_path/'new.out' + link = files.os.link + def race(src, dst): + path.write_bytes(b'External creation') + link(src, dst) + monkeypatch.setattr(files.os, 'link', race) + with pytest.raises(FileExistsError): files.atomic_write(path, b'Export', expected=None) + assert path.read_bytes() == b'External creation' + + +def test_second_conflict_during_recovery_keeps_both_external_versions(tmp_path, monkeypatch): + path = tmp_path/'existing.out'; path.write_bytes(b'Original') + expected = files.fingerprint(path) + replace = files._replace_with_backup + count = 0 + def racing_replace(temporary, target, backup): + nonlocal count + count += 1; path.write_bytes(f'External {count}'.encode()) + replace(temporary, target, backup) + monkeypatch.setattr(files, '_replace_with_backup', racing_replace) + with pytest.raises(OSError, match='恢复副本'): files.atomic_write(path, b'Export', expected=expected) + recovered = [p.read_bytes() for p in tmp_path.iterdir()] + assert b'External 1' in recovered and b'External 2' in recovered + + +@pytest.mark.parametrize('engine', ['builtin', 'python-docx', 'pandoc']) +def test_table_first_list_preserves_cells_and_numbered_empty_paragraph(tmp_path, engine): + path = export_markdown_to_docx(TABLE_LIST, tmp_path/'table.docx', engine=engine) + doc, _ = parts(path) + body = doc.find(W+'body'); table = body.find(W+'tbl') + assert [''.join(c.itertext()) for c in table.findall('w:tr/w:tc', NS)] == ['HeadOne','HeadTwo','CellOne','CellTwo'] + before = table.getprevious() + assert before.tag == W+'p' and before.find('w:pPr/w:numPr', NS) is not None + continued = next(p for p in doc.iter(W+'p') if ''.join(p.itertext()) == 'ContinuedText') + num = continued.find('w:pPr/w:numPr/w:numId', NS) + if num is not None: + with zipfile.ZipFile(path) as package: numbering = ET.fromstring(package.read('word/numbering.xml')) + abstract = numbering.find(f"w:num[@w:numId='{num.get(W+'val')}']/w:abstractNumId", NS).get(W+'val') + marker = numbering.find(f"w:abstractNum[@w:abstractNumId='{abstract}']/w:lvl/w:lvlText", NS).get(W+'val') + assert not marker.strip() + assert '\u200b' not in ''.join(doc.itertext()) + + +@pytest.mark.parametrize('prefix', ['1. ', '- ', '> 1. ', '1. > ']) +def test_combined_heading_preserves_container_prefix(tmp_path, prefix): + source = prefix + '### Heading\n' + combined = batch.combine_documents([batch.BatchDocument(tmp_path/'one.md', source)], MarkdownRenderer(tmp_path)) + assert prefix + '#### Heading {#mdh1h1}' in combined.word_source + + +def test_combined_word_keeps_heading_numbering(tmp_path): + combined = batch.combine_documents([batch.BatchDocument(tmp_path/'one.md', HEADING_LIST)], MarkdownRenderer(tmp_path)) + doc, _ = parts(export_markdown_to_docx(combined.word_source, tmp_path/'combined.docx', engine='pandoc')) + headings = [p for p in doc.iter(W+'p') if ''.join(p.itertext()) in ('FirstHeading', 'LastHeading')] + assert len(headings) == 2 + assert all(p.find('w:pPr/w:numPr', NS) is not None for p in headings) diff --git a/tests/test_interaction_revision_056.py b/tests/test_interaction_revision_056.py index f4e8d4f..43aa222 100644 --- a/tests/test_interaction_revision_056.py +++ b/tests/test_interaction_revision_056.py @@ -15,8 +15,8 @@ def test_batch_preserves_external_changes_between_writes(tmp_path, monkeypatch, if changed != 'created': second.write_bytes(b'original') expected = {p: batch.fingerprint(p) for p in (first, second)} write = batch.atomic_write - def concurrent_write(path, data): - write(path, data) + def concurrent_write(path, data, **kwargs): + write(path, data, **kwargs) if path == first and data == b'exported': second.write_bytes(b'external second') if changed == 'rollback': first.write_bytes(b'external first') @@ -35,8 +35,8 @@ def test_rename_preserves_external_edits_during_transaction(tmp_path, monkeypatc ref.write_text('[ref](old.md)', encoding='utf8') plan = rename.plan_rename(tmp_path, source, target) write = rename.atomic_write - def concurrent_write(path, data): - write(path, data) + def concurrent_write(path, data, **kwargs): + write(path, data, **kwargs) if (when == 'backup' and path.name == 'manifest.json') or (when != 'backup' and path == target): ref.write_bytes(b'external ref') if when == 'rollback': target.write_bytes(b'external source') diff --git a/tests/test_rename_references.py b/tests/test_rename_references.py index 1967e2c..2fa26e5 100644 --- a/tests/test_rename_references.py +++ b/tests/test_rename_references.py @@ -32,9 +32,9 @@ def test_changed_disk_aborts_and_partial_write_rolls_back(tmp_path, monkeypatch) old, new, ref = fixture_documents(tmp_path) plan = renaming.plan_rename(tmp_path, old, new) write = renaming.atomic_write - def fail(path, data): + def fail(path, data, **kwargs): if path == ref: raise OSError('simulated permission error') - write(path, data) + write(path, data, **kwargs) monkeypatch.setattr(renaming, 'atomic_write', fail) with pytest.raises(OSError, match='文件已恢复'): renaming.execute_rename(plan, tmp_path / 'backups') assert not new.exists() diff --git a/tests/word_revision_057_smoke.ps1 b/tests/word_revision_057_smoke.ps1 new file mode 100644 index 0000000..7e9d7f2 --- /dev/null +++ b/tests/word_revision_057_smoke.ps1 @@ -0,0 +1,48 @@ +param([string]$Python = (Join-Path $PSScriptRoot '..\mdhelper_env\mdhelper_env\Scripts\python.exe')) +$ErrorActionPreference='Stop' +Push-Location (Join-Path $PSScriptRoot '..') +$word=$null +try { + $folder = (@' +import sys,tempfile +from pathlib import Path +sys.path.insert(0,str(Path('tests').resolve())) +from test_content_revision_057 import * +folder=Path(tempfile.mkdtemp(prefix='MDHelper-057-Word-')) +for engine in ('builtin','python-docx','pandoc'): + export_markdown_to_docx(TABLE_LIST,folder/f'{engine}-table.docx',engine=engine) +combined=batch.combine_documents([batch.BatchDocument(folder/'chapter.md',HEADING_LIST)],MarkdownRenderer(folder)) +export_markdown_to_docx(combined.word_source,folder/'pandoc-combined.docx',engine='pandoc') +print(folder) +'@ | & $Python -B -).Trim() + if ($LASTEXITCODE -ne 0) {throw 'Fixture generation failed'} + $word=New-Object -ComObject Word.Application + $word.Visible=$false; $word.DisplayAlerts=0 + foreach($engine in @('builtin','python-docx','pandoc')) { + $doc=$word.Documents.Open((Join-Path $folder "$engine-table.docx"),$false,$true,$false) + try { + foreach($text in @('HeadOne','HeadTwo','CellOne','CellTwo')) { + $r=$doc.Content.Duplicate + if(-not $r.Find.Execute($text)){throw "$engine lost table text $text"} + } + if($doc.Paragraphs.Item(1).Range.ListFormat.ListString.Trim() -ne '1.'){throw "$engine table marker missing"} + $r=$doc.Content.Duplicate; [void]$r.Find.Execute('ContinuedText') + if($r.ListFormat.ListString.Trim() -ne ''){throw "$engine continuation gained number"} + $r=$doc.Content.Duplicate; [void]$r.Find.Execute('LastText') + if($r.ListFormat.ListString.Trim() -ne '2.'){throw "$engine numbering restarted"} + Write-Output "$engine table content and numbering PASS" + } finally {$doc.Close(0)} + } + $doc=$word.Documents.Open((Join-Path $folder 'pandoc-combined.docx'),$false,$true,$false) + try { + foreach($pair in @(@('FirstHeading','1.'),@('ContinuedText',''),@('LastHeading','2.'))) { + $r=$doc.Content.Duplicate;[void]$r.Find.Execute($pair[0]) + if($r.ListFormat.ListString.Trim() -ne $pair[1]){throw "combined heading marker wrong: $($pair[0])"} + } + Write-Output 'combined headings PASS' + } finally {$doc.Close(0)} + Write-Output "Fixtures: $folder" +} finally { + if($word){$word.Quit();[void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($word)} + Pop-Location +}