154 lines
4.8 KiB
Python
154 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Iterator
|
|
|
|
from .media import MARKDOWN_EXTENSIONS, WEBPAGE_EXTENSIONS
|
|
from .markdown_utils import extract_headings_from_text
|
|
from .text_io import DEFAULT_TEXT_PREVIEW_BYTES, is_within_size_limit, read_text_with_fallback
|
|
|
|
PYTHON_EXTENSIONS = {".py", ".pyw", ".pyi"}
|
|
CPP_EXTENSIONS = {".c", ".cc", ".cxx", ".cpp", ".h", ".hh", ".hxx", ".hpp"}
|
|
|
|
FILE_GROUP_EXTENSIONS = {
|
|
"markdown": MARKDOWN_EXTENSIONS | WEBPAGE_EXTENSIONS,
|
|
"python": PYTHON_EXTENSIONS,
|
|
"cpp": CPP_EXTENSIONS,
|
|
}
|
|
DEFAULT_DOCUMENT_FILE_GROUPS = ("markdown",)
|
|
SUPPORTED_DOCUMENT_EXTENSIONS = set().union(*FILE_GROUP_EXTENSIONS.values())
|
|
CODE_DOCUMENT_EXTENSIONS = PYTHON_EXTENSIONS | CPP_EXTENSIONS
|
|
MAX_DISCOVERABLE_CODE_BYTES = DEFAULT_TEXT_PREVIEW_BYTES
|
|
|
|
EXCLUDED_DIR_NAMES = {
|
|
".git",
|
|
"__pycache__",
|
|
".pytest_cache",
|
|
".mypy_cache",
|
|
"build",
|
|
"dist",
|
|
"logs",
|
|
"html",
|
|
"mdhelper_env",
|
|
"vendor",
|
|
"venv",
|
|
".venv",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DocumentEntry:
|
|
path: Path
|
|
title: str
|
|
relative_path: str
|
|
|
|
|
|
def normalized_document_file_groups(groups: object) -> tuple[str, ...]:
|
|
if isinstance(groups, str):
|
|
candidates = [part.strip() for part in groups.split(",")]
|
|
elif isinstance(groups, (list, tuple, set)):
|
|
candidates = [str(part).strip() for part in groups]
|
|
else:
|
|
candidates = list(DEFAULT_DOCUMENT_FILE_GROUPS)
|
|
|
|
normalized: list[str] = []
|
|
for group in candidates:
|
|
if group in FILE_GROUP_EXTENSIONS and group not in normalized:
|
|
normalized.append(group)
|
|
return tuple(normalized or DEFAULT_DOCUMENT_FILE_GROUPS)
|
|
|
|
|
|
def document_extensions_for_groups(groups: object = None) -> set[str]:
|
|
enabled_groups = normalized_document_file_groups(groups)
|
|
extensions: set[str] = set()
|
|
for group in enabled_groups:
|
|
extensions.update(FILE_GROUP_EXTENSIONS[group])
|
|
return extensions
|
|
|
|
|
|
def is_code_document(path: Path) -> bool:
|
|
return path.suffix.lower() in CODE_DOCUMENT_EXTENSIONS
|
|
|
|
|
|
def code_language_for_path(path: Path) -> str:
|
|
suffix = path.suffix.lower()
|
|
if suffix in PYTHON_EXTENSIONS:
|
|
return "python"
|
|
if suffix in {".c", ".h"}:
|
|
return "c"
|
|
if suffix in CPP_EXTENSIONS:
|
|
return "cpp"
|
|
return "text"
|
|
|
|
|
|
def is_discoverable_document(path: Path, supported_extensions: set[str]) -> bool:
|
|
suffix = path.suffix.lower()
|
|
if suffix not in supported_extensions:
|
|
return False
|
|
if suffix in CODE_DOCUMENT_EXTENSIONS:
|
|
return is_within_size_limit(path, MAX_DISCOVERABLE_CODE_BYTES)
|
|
return True
|
|
|
|
|
|
def first_heading(path: Path) -> str | None:
|
|
if path.suffix.lower() not in {".md", ".markdown"}:
|
|
return None
|
|
text = read_text_with_fallback(path, max_bytes=DEFAULT_TEXT_PREVIEW_BYTES)
|
|
if not text or text.startswith("读取失败:"):
|
|
return None
|
|
headings = extract_headings_from_text(text)
|
|
return headings[0].title if headings else None
|
|
|
|
|
|
def discover_documents(project_root: Path, enabled_groups: object = None) -> list[DocumentEntry]:
|
|
root = project_root.resolve()
|
|
supported_extensions = document_extensions_for_groups(enabled_groups)
|
|
entries: list[DocumentEntry] = []
|
|
|
|
for path in _iter_project_files(root):
|
|
if not is_discoverable_document(path, supported_extensions):
|
|
continue
|
|
try:
|
|
relative_path = path.relative_to(root).as_posix()
|
|
except ValueError:
|
|
continue
|
|
resolved = path.resolve()
|
|
title = first_heading(resolved) or resolved.stem.replace("_", " ").title()
|
|
entries.append(DocumentEntry(path=resolved, title=title, relative_path=relative_path))
|
|
|
|
return entries
|
|
|
|
|
|
def _iter_project_files(root: Path) -> Iterator[Path]:
|
|
stack = [root]
|
|
while stack:
|
|
directory = stack.pop()
|
|
try:
|
|
with os.scandir(directory) as scanner:
|
|
dirs: list[tuple[str, Path]] = []
|
|
files: list[tuple[str, Path]] = []
|
|
for entry in scanner:
|
|
name = entry.name
|
|
if _is_excluded_name(name):
|
|
continue
|
|
try:
|
|
if entry.is_dir(follow_symlinks=False):
|
|
dirs.append((name.casefold(), Path(entry.path)))
|
|
elif entry.is_file(follow_symlinks=False):
|
|
files.append((name.casefold(), Path(entry.path)))
|
|
except OSError:
|
|
continue
|
|
except OSError:
|
|
continue
|
|
|
|
for _, path in sorted(files):
|
|
yield path
|
|
for _, path in reversed(sorted(dirs)):
|
|
stack.append(path)
|
|
|
|
|
|
def _is_excluded_name(name: str) -> bool:
|
|
return name.startswith(".") or name in EXCLUDED_DIR_NAMES
|