fix: retain conflicting saves and preserve list content in Word

This commit is contained in:
2026-11-05 11:57:06 +08:00
parent e605c2f8d3
commit b3a0907e1a
11 changed files with 283 additions and 27 deletions
+5 -13
View File
@@ -11,7 +11,7 @@ from urllib.parse import unquote, urlsplit, urlunsplit
from .document_resources import analyze_document, apply_source_edits, resolve_local_reference, SourceEdit
from .doc_check import document_anchors
from .export_jobs import check_cancelled
from .file_versions import atomic_write
from .file_versions import atomic_write, fingerprint, checked_remove
from .markdown_utils import Heading
@@ -88,7 +88,7 @@ def combine_documents(documents: list[BatchDocument], renderer, cancel=None) ->
content = tokens[index + 1].content
headings.append(Heading(level, content, anchor))
start, end = token.map
prefix = re.match(r"[ \t]*(?:>[ \t]*)*", word_source[offsets[start]:]).group()
prefix = re.match(r"[ \t]*(?:>[ \t]*|(?:[-+*]|\d+[.)])[ \t]+)*", word_source[offsets[start]:]).group()
word_edits.append(SourceEdit(offsets[start], offsets[end], f"{prefix}{'#' * level} {next(word_heading_texts)} {{#{anchor}}}\n"))
elif token.type == "heading_close":
token.tag = f"h{min(6, int(token.tag[1:]) + 1)}"
@@ -107,14 +107,6 @@ def combine_documents(documents: list[BatchDocument], renderer, cancel=None) ->
return CombinedDocument("\n\n".join(word), "\n".join(bodies), headings)
def fingerprint(path: Path):
if not path.exists(): return None
digest = hashlib.sha256()
with path.open('rb') as stream:
for block in iter(lambda: stream.read(1024 * 1024), b''): digest.update(block)
return digest.hexdigest()
def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, cancel=None):
"""Publish only after all conversions succeed; restore earlier outputs on any failure."""
check_cancelled(cancel)
@@ -140,7 +132,7 @@ def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, can
# destination again immediately before publishing this particular file.
if fingerprint(destination) != expected[destination]:
raise OSError(f"导出期间目标文件发生变化,已停止替换:{destination}")
atomic_write(destination, data)
atomic_write(destination, data, expected=expected[destination])
written.append((destination, hashlib.sha256(data).hexdigest()))
check_cancelled(cancel)
except Exception as error:
@@ -149,8 +141,8 @@ def publish_outputs(outputs: dict[Path, Path], expected: dict, backup: Path, can
try:
if fingerprint(destination) != published:
raise OSError(f"目标文件已被外部修改,保留当前内容:{destination}")
if originals[destination] is None: destination.unlink(missing_ok=True)
else: atomic_write(destination, originals[destination].read_bytes())
if originals[destination] is None: checked_remove(destination, published)
else: atomic_write(destination, originals[destination].read_bytes(), expected=published)
except OSError as rollback:
failures.append(str(rollback))
if failures:
+21 -1
View File
@@ -141,7 +141,7 @@ def export_markdown_to_docx_pandoc(source: str, output_path: Path, source_path:
# Export the current editor text, including unsaved edits. Retain the source
# directory only for resolving linked images and other local resources.
with tempfile.NamedTemporaryFile("w", suffix=".md", encoding="utf-8", delete=False) as handle:
handle.write(source)
handle.write(_pandoc_list_tables(source))
temporary_path = Path(handle.name)
input_path = temporary_path
resource_directory = source_path.resolve().parent if source_path is not None else input_path.parent
@@ -233,6 +233,24 @@ def _markdown_blocks(source: str) -> list[ParagraphBlock | TableBlock]:
return _DocxBuilder(markdown.parse(source)).build()
def _pandoc_list_tables(source: str) -> str:
"""Disambiguate table-first list items for Pandoc's Markdown reader."""
markdown = MarkdownIt('commonmark').enable('table')
lines = source.splitlines(keepends=True)
items = []
for token in markdown.parse(source):
if token.type == 'list_item_open': items.append(token.map[0])
elif token.type == 'list_item_close': items.pop()
elif token.type == 'table_open' and items and token.map[0] == items[-1]:
row = token.map[0]
match = re.match(r'([ \t]*(?:>[ \t]*)*)([-+*]|\d+[.)])([ \t]+)', lines[row])
if match:
prefix, marker, spacing = match.groups()
indent = prefix + ' ' * (len(marker) + len(spacing))
lines[row] = match.group() + '\n' + prefix + '\n' + indent + lines[row][match.end():]
return ''.join(lines)
def _add_python_docx_paragraph(document: object, block: ParagraphBlock) -> None:
style = f"Heading {block.style.removeprefix('Heading')}" if block.style.startswith("Heading") else None
special = block.style.startswith('Heading') or block.style == 'Code'
@@ -398,6 +416,8 @@ class _DocxBuilder:
self.list_starts.pop()
self.list_quote_depths.pop()
elif token.type == "table_open":
if self.item_pending and self.item_pending[-1]:
self._append_paragraph([], 'Normal', self.list_stack[-1])
self.blocks.append(self._collect_table())
continue
elif token.type == "hr":
+4 -1
View File
@@ -509,6 +509,9 @@ class WordStyles:
ppr = properties(paragraph, 'pPr')
original_id = reference(ppr, 'pStyle') or 'Normal'
names = {reference(style, 'name') for style in self.chain(original_id)}
if 'MDHelperSourceTableMarker' in names:
for text in paragraph.iter(W + 't'):
if text.text == '\u200b': text.text = ''
for name in tuple(names):
if name in SOURCE_LIST_STYLES:
base, list_target = SOURCE_LIST_STYLES[name]
@@ -584,7 +587,7 @@ def apply_semantic_styles(path, rules, **defaults):
if any(source_style_order(reference(s, 'name')) for s in adapter.chain(identifier)):
adapter.apply_run(run, 'paragraph')
for style in list(parts['word/styles.xml']):
if source_style_order(reference(style, 'name')) or reference(style, 'name') in SOURCE_PARAGRAPH_STYLES:
if source_style_order(reference(style, 'name')) or reference(style, 'name') in (*SOURCE_PARAGRAPH_STYLES, 'MDHelperSourceTableMarker'):
parts['word/styles.xml'].remove(style)
for filename, root in parts.items():
_order_properties(root)
+87 -2
View File
@@ -8,8 +8,90 @@ import time
import uuid
from pathlib import Path
_UNCHECKED = object()
def atomic_write(path: Path, data: bytes) -> None:
def fingerprint(path: Path):
try:
with path.open('rb') as stream:
return hashlib.file_digest(stream, 'sha256').hexdigest()
except FileNotFoundError:
return None
def _replace_with_backup(temporary: Path, path: Path, backup: Path) -> None:
"""Retain the exact file displaced at the commit boundary, not an earlier copy."""
if os.name != 'nt':
# Never silently fall back to an unchecked overwrite on another platform.
raise OSError('当前平台不支持受保护的文件替换。')
import ctypes
from ctypes import wintypes
replace = ctypes.WinDLL('kernel32', use_last_error=True).ReplaceFileW
replace.argtypes = [wintypes.LPCWSTR] * 3 + [wintypes.DWORD, wintypes.LPVOID, wintypes.LPVOID]
replace.restype = wintypes.BOOL
if not replace(str(path), str(temporary), str(backup), 0, None, None):
raise ctypes.WinError(ctypes.get_last_error())
def _recovery_path(path: Path) -> Path:
return path.with_name(f'.{path.name}.conflict-{uuid.uuid4().hex}.bak')
def _checked_replace(temporary: Path, path: Path, expected: str | None) -> None:
if fingerprint(path) != expected:
raise OSError(f'文件发生变化,已停止替换:{path}')
if expected is None:
# Creating the directory entry must fail if another process got there first.
os.link(temporary, path)
return
backup = _recovery_path(path)
try:
published = fingerprint(temporary)
_replace_with_backup(temporary, path, backup)
if fingerprint(backup) == expected:
try:
backup.unlink()
except OSError:
pass # A retained backup must not turn a committed write into failure.
return
# Retain the conflicting version even if another process saves again
# during recovery. Every displaced file has its own recovery name.
if fingerprint(path) == published:
descriptor, restore_name = tempfile.mkstemp(dir=path.parent, suffix='.tmp')
restore = Path(restore_name)
try:
with os.fdopen(descriptor, 'wb') as stream:
stream.write(backup.read_bytes())
displaced = _recovery_path(path)
_replace_with_backup(restore, path, displaced)
if fingerprint(displaced) == published:
displaced.unlink()
finally:
restore.unlink(missing_ok=True)
raise OSError(f'文件在替换时发生变化,已保留冲突版本:{backup}')
except OSError as exc:
if backup.exists():
raise OSError(f'文件替换未完成,恢复副本位于 {backup}:{exc}') from exc
raise
def checked_remove(path: Path, expected: str) -> None:
"""Rollback a newly created output without deleting a concurrent save."""
if fingerprint(path) != expected:
raise OSError(f'文件已被外部修改,保留当前内容:{path}')
recovery = _recovery_path(path)
path.rename(recovery)
if fingerprint(recovery) == expected:
recovery.unlink()
else:
try:
os.link(recovery, path) # Exclusive restore; never overwrite a newer file.
except OSError:
pass
raise OSError(f'文件在恢复时发生变化,保留副本:{recovery}')
def atomic_write(path: Path, data: bytes, *, expected=_UNCHECKED) -> None:
path = path.resolve()
path.parent.mkdir(parents=True, exist_ok=True)
descriptor, temporary = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
@@ -18,7 +100,10 @@ def atomic_write(path: Path, data: bytes) -> None:
stream.write(data)
stream.flush()
os.fsync(stream.fileno())
os.replace(temporary, path)
if expected is _UNCHECKED:
os.replace(temporary, path)
else:
_checked_replace(Path(temporary), path, expected)
finally:
Path(temporary).unlink(missing_ok=True)
+3 -2
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import difflib
import hashlib
import json
import os
import time
@@ -91,7 +92,7 @@ def execute_rename(plan: RenamePlan, backup_root: Path | None = None) -> Path:
destination = plan.target if path == plan.source else path
if destination.read_bytes() != plan.originals[path]:
raise OSError(f"文件在重命名期间已变化,保留当前内容:{destination.name}")
atomic_write(destination, data)
atomic_write(destination, data, expected=hashlib.sha256(plan.originals[path]).hexdigest())
written.append(path)
except OSError as exc:
failures = []
@@ -100,7 +101,7 @@ def execute_rename(plan: RenamePlan, backup_root: Path | None = None) -> Path:
destination = plan.target if path == plan.source else path
if destination.read_bytes() != plan.replacements[path]:
raise OSError(f"文件已被外部修改,保留当前内容:{destination.name}")
atomic_write(destination, plan.originals[path])
atomic_write(destination, plan.originals[path], expected=hashlib.sha256(plan.replacements[path]).hexdigest())
except OSError as rollback_error:
failures.append(str(rollback_error))
if renamed:
@@ -48,6 +48,12 @@ local function mark(items, stack)
end
local function list_paragraphs(list)
for _, item in ipairs(list.content) do
-- Word numbers paragraphs, not tables. Give a table-first item its own
-- empty numbered paragraph; its following text remains a continuation.
if item[1] and item[1].t == 'Table' then
item:insert(1, pandoc.Div({pandoc.Para({pandoc.Str('\u{200b}')})},
pandoc.Attr('', {}, {['custom-style']='MDHelperSourceTableMarker'})))
end
local target = list.t == 'OrderedList' and 'list_ordered' or 'list_bullet'
local text = pandoc.utils.stringify(item[1] or {})
if text:match('^%[[ xX]%]') or text:sub(1, #'☐') == '☐' or text:sub(1, #'☒') == '☒' or text:sub(1, #'☑') == '☑' then
+2 -2
View File
@@ -53,9 +53,9 @@ def test_publication_cancellation_race_and_rollback(tmp_path, monkeypatch):
with pytest.raises(OSError, match='发生变化'): publish_outputs(outputs, expected, tmp_path)
destinations[0].write_bytes(b'old')
real_write = batch_documents.atomic_write
def fail(path, data):
def fail(path, data, **kwargs):
if path == destinations[1] and data == b'new': raise OSError('disk full')
real_write(path, data)
real_write(path, data, **kwargs)
monkeypatch.setattr(batch_documents, 'atomic_write', fail)
with pytest.raises(OSError, match='disk full'): publish_outputs(outputs, expected, tmp_path)
assert all(path.read_bytes() == b'old' for path in destinations)
+101
View File
@@ -0,0 +1,101 @@
"""Commit-boundary conflicts and preservation of list content in Word."""
import zipfile
from lxml import etree as ET
import pytest
from mdhelper import file_versions as files, batch_documents as batch, rename_references as rename
from mdhelper.docx_exporter import export_markdown_to_docx
from mdhelper.markdown_renderer import MarkdownRenderer
from mdhelper.docx_style import W, NS
from test_style_regressions import parts
TABLE_LIST = '1. | HeadOne | HeadTwo |\n |---|---|\n | CellOne | CellTwo |\n\n ContinuedText\n\n2. LastText'
HEADING_LIST = '1. ### FirstHeading\n\n ContinuedText\n\n2. ### LastHeading'
@pytest.mark.parametrize('operation', ['batch', 'rename'])
@pytest.mark.parametrize('boundary', ['flush', 'replace'])
def test_transactions_retain_external_save_at_commit(tmp_path, monkeypatch, operation, boundary):
target = tmp_path/'ref.md'
target.write_bytes(b'[Link](old.md)')
stage = tmp_path/'stage'; stage.write_bytes(b'Exported')
source = tmp_path/'old.md'; source.write_bytes(b'Source')
plan = rename.plan_rename(tmp_path, source, tmp_path/'new.md')
expected = {target: files.fingerprint(target)}
replace, fsync = files._replace_with_backup, files.os.fsync
external = b'External save at final commit boundary'
injected = False
def race_replace(temporary, path, backup):
nonlocal injected
if path == target and not injected:
injected = True; target.write_bytes(external)
replace(temporary, path, backup)
def race_flush(descriptor):
nonlocal injected
fsync(descriptor)
if not injected:
injected = True; target.write_bytes(external)
monkeypatch.setattr(files, '_replace_with_backup', race_replace) if boundary == 'replace' else monkeypatch.setattr(files.os, 'fsync', race_flush)
with pytest.raises(OSError):
if operation == 'batch': batch.publish_outputs({target: stage}, expected, tmp_path)
else: rename.execute_rename(plan, tmp_path/'backups')
assert target.read_bytes() == external
assert source.exists() and not (tmp_path/'new.md').exists()
def test_creation_does_not_overwrite_file_created_at_commit(tmp_path, monkeypatch):
path = tmp_path/'new.out'
link = files.os.link
def race(src, dst):
path.write_bytes(b'External creation')
link(src, dst)
monkeypatch.setattr(files.os, 'link', race)
with pytest.raises(FileExistsError): files.atomic_write(path, b'Export', expected=None)
assert path.read_bytes() == b'External creation'
def test_second_conflict_during_recovery_keeps_both_external_versions(tmp_path, monkeypatch):
path = tmp_path/'existing.out'; path.write_bytes(b'Original')
expected = files.fingerprint(path)
replace = files._replace_with_backup
count = 0
def racing_replace(temporary, target, backup):
nonlocal count
count += 1; path.write_bytes(f'External {count}'.encode())
replace(temporary, target, backup)
monkeypatch.setattr(files, '_replace_with_backup', racing_replace)
with pytest.raises(OSError, match='恢复副本'): files.atomic_write(path, b'Export', expected=expected)
recovered = [p.read_bytes() for p in tmp_path.iterdir()]
assert b'External 1' in recovered and b'External 2' in recovered
@pytest.mark.parametrize('engine', ['builtin', 'python-docx', 'pandoc'])
def test_table_first_list_preserves_cells_and_numbered_empty_paragraph(tmp_path, engine):
path = export_markdown_to_docx(TABLE_LIST, tmp_path/'table.docx', engine=engine)
doc, _ = parts(path)
body = doc.find(W+'body'); table = body.find(W+'tbl')
assert [''.join(c.itertext()) for c in table.findall('w:tr/w:tc', NS)] == ['HeadOne','HeadTwo','CellOne','CellTwo']
before = table.getprevious()
assert before.tag == W+'p' and before.find('w:pPr/w:numPr', NS) is not None
continued = next(p for p in doc.iter(W+'p') if ''.join(p.itertext()) == 'ContinuedText')
num = continued.find('w:pPr/w:numPr/w:numId', NS)
if num is not None:
with zipfile.ZipFile(path) as package: numbering = ET.fromstring(package.read('word/numbering.xml'))
abstract = numbering.find(f"w:num[@w:numId='{num.get(W+'val')}']/w:abstractNumId", NS).get(W+'val')
marker = numbering.find(f"w:abstractNum[@w:abstractNumId='{abstract}']/w:lvl/w:lvlText", NS).get(W+'val')
assert not marker.strip()
assert '\u200b' not in ''.join(doc.itertext())
@pytest.mark.parametrize('prefix', ['1. ', '- ', '> 1. ', '1. > '])
def test_combined_heading_preserves_container_prefix(tmp_path, prefix):
source = prefix + '### Heading\n'
combined = batch.combine_documents([batch.BatchDocument(tmp_path/'one.md', source)], MarkdownRenderer(tmp_path))
assert prefix + '#### Heading {#mdh1h1}' in combined.word_source
def test_combined_word_keeps_heading_numbering(tmp_path):
combined = batch.combine_documents([batch.BatchDocument(tmp_path/'one.md', HEADING_LIST)], MarkdownRenderer(tmp_path))
doc, _ = parts(export_markdown_to_docx(combined.word_source, tmp_path/'combined.docx', engine='pandoc'))
headings = [p for p in doc.iter(W+'p') if ''.join(p.itertext()) in ('FirstHeading', 'LastHeading')]
assert len(headings) == 2
assert all(p.find('w:pPr/w:numPr', NS) is not None for p in headings)
+4 -4
View File
@@ -15,8 +15,8 @@ def test_batch_preserves_external_changes_between_writes(tmp_path, monkeypatch,
if changed != 'created': second.write_bytes(b'original')
expected = {p: batch.fingerprint(p) for p in (first, second)}
write = batch.atomic_write
def concurrent_write(path, data):
write(path, data)
def concurrent_write(path, data, **kwargs):
write(path, data, **kwargs)
if path == first and data == b'exported':
second.write_bytes(b'external second')
if changed == 'rollback': first.write_bytes(b'external first')
@@ -35,8 +35,8 @@ def test_rename_preserves_external_edits_during_transaction(tmp_path, monkeypatc
ref.write_text('[ref](old.md)', encoding='utf8')
plan = rename.plan_rename(tmp_path, source, target)
write = rename.atomic_write
def concurrent_write(path, data):
write(path, data)
def concurrent_write(path, data, **kwargs):
write(path, data, **kwargs)
if (when == 'backup' and path.name == 'manifest.json') or (when != 'backup' and path == target):
ref.write_bytes(b'external ref')
if when == 'rollback': target.write_bytes(b'external source')
+2 -2
View File
@@ -32,9 +32,9 @@ def test_changed_disk_aborts_and_partial_write_rolls_back(tmp_path, monkeypatch)
old, new, ref = fixture_documents(tmp_path)
plan = renaming.plan_rename(tmp_path, old, new)
write = renaming.atomic_write
def fail(path, data):
def fail(path, data, **kwargs):
if path == ref: raise OSError('simulated permission error')
write(path, data)
write(path, data, **kwargs)
monkeypatch.setattr(renaming, 'atomic_write', fail)
with pytest.raises(OSError, match='文件已恢复'): renaming.execute_rename(plan, tmp_path / 'backups')
assert not new.exists()
+48
View File
@@ -0,0 +1,48 @@
param([string]$Python = (Join-Path $PSScriptRoot '..\mdhelper_env\mdhelper_env\Scripts\python.exe'))
$ErrorActionPreference='Stop'
Push-Location (Join-Path $PSScriptRoot '..')
$word=$null
try {
$folder = (@'
import sys,tempfile
from pathlib import Path
sys.path.insert(0,str(Path('tests').resolve()))
from test_content_revision_057 import *
folder=Path(tempfile.mkdtemp(prefix='MDHelper-057-Word-'))
for engine in ('builtin','python-docx','pandoc'):
export_markdown_to_docx(TABLE_LIST,folder/f'{engine}-table.docx',engine=engine)
combined=batch.combine_documents([batch.BatchDocument(folder/'chapter.md',HEADING_LIST)],MarkdownRenderer(folder))
export_markdown_to_docx(combined.word_source,folder/'pandoc-combined.docx',engine='pandoc')
print(folder)
'@ | & $Python -B -).Trim()
if ($LASTEXITCODE -ne 0) {throw 'Fixture generation failed'}
$word=New-Object -ComObject Word.Application
$word.Visible=$false; $word.DisplayAlerts=0
foreach($engine in @('builtin','python-docx','pandoc')) {
$doc=$word.Documents.Open((Join-Path $folder "$engine-table.docx"),$false,$true,$false)
try {
foreach($text in @('HeadOne','HeadTwo','CellOne','CellTwo')) {
$r=$doc.Content.Duplicate
if(-not $r.Find.Execute($text)){throw "$engine lost table text $text"}
}
if($doc.Paragraphs.Item(1).Range.ListFormat.ListString.Trim() -ne '1.'){throw "$engine table marker missing"}
$r=$doc.Content.Duplicate; [void]$r.Find.Execute('ContinuedText')
if($r.ListFormat.ListString.Trim() -ne ''){throw "$engine continuation gained number"}
$r=$doc.Content.Duplicate; [void]$r.Find.Execute('LastText')
if($r.ListFormat.ListString.Trim() -ne '2.'){throw "$engine numbering restarted"}
Write-Output "$engine table content and numbering PASS"
} finally {$doc.Close(0)}
}
$doc=$word.Documents.Open((Join-Path $folder 'pandoc-combined.docx'),$false,$true,$false)
try {
foreach($pair in @(@('FirstHeading','1.'),@('ContinuedText',''),@('LastHeading','2.'))) {
$r=$doc.Content.Duplicate;[void]$r.Find.Execute($pair[0])
if($r.ListFormat.ListString.Trim() -ne $pair[1]){throw "combined heading marker wrong: $($pair[0])"}
}
Write-Output 'combined headings PASS'
} finally {$doc.Close(0)}
Write-Output "Fixtures: $folder"
} finally {
if($word){$word.Quit();[void][Runtime.InteropServices.Marshal]::FinalReleaseComObject($word)}
Pop-Location
}