1"""Multi-line needs in source comments: comment runs, prefix stripping, the block grammar.
2
3A multi-line need is one need written across the lines of a comment. This module turns
4the comment nodes of one file into comment RUNS, strips each line's comment prefix by the
5comment's kind, and parses a run's lines with the block grammar. It knows nothing of
6Sphinx or of a need's fields: the record is built by ``analyse.py``, and the need is
7validated and its content parsed by whoever consumes the record.
8
9**Runs.** One block comment (``/* … */``, ``/** … */``, ``/*! … */``) or one Python string
10statement (a docstring or any bare string statement) is a run on its own. Consecutive line comments form one run when they sit on consecutive rows,
11use the same delimiter, and have only whitespace before them on their rows; a line
12comment after code on its row neither starts nor continues a run.
13
14**Prefix stripping** follows the comment's kind and never searches the text, so each
15logical line maps to exactly one source row:
16
17======================================== ================================================== ===========================================
18comment kind stripped from each line languages
19======================================== ================================================== ===========================================
20line comment ``//``, ``///``, ``//!`` the delimiter, then ONE space if present C/C++, C#, Rust, Go, JSONC
21line comment ``#`` the delimiter, then ONE space if present Python, YAML, Bash
22block comment ``/* … */``, ``/** … */``, a first row holding only the opener (``/*`` with C/C++, C#, Rust, Go, JSONC
23``/*! … */`` any stars, or ``/*!``) and a last row holding only
24 the closer (stars then ``*/``) are dropped; the
25 delimiters are removed; then a ``*`` leader (``*``
26 and one space, or a lone ``*``) only if EVERY
27 remaining non-blank line after the opener's row
28 carries one (the doxygen rule)
29Python docstring the quotes and any string prefix; the interior Python
30 dedented as ``inspect.cleandoc`` does (no line is
31 removed)
32line comment ``--`` one row to add when the language lands VHDL, SQL (sphinx-needs #2077)
33block comment ``<!-- … -->`` one row to add when the language lands Markdown (sphinx-needs #1895)
34======================================== ================================================== ===========================================
35
36A comment whose text does not start with a delimiter its kind allows -- impossible for a
37tree-sitter node, possible for a libclang one -- keeps its lines unstripped, and a debug
38message says so.
39
40**The grammar** (after stripping) is the one in ``docs/components/analyse.rst``: the open
41line ``<open>[<markup>] <type>: <title>``, option lines ``:key: value`` (a line indented
42deeper than its option line continues the value), the body in the declared markup, and the
43close word alone on its line. :func:`parse_run` returns the blocks, the refusals and the
44rows a block claims; the claimed rows are hidden from the one-line and ``@need-ids:``
45extractors.
46"""
47
48from __future__ import annotations
49
50import re
51from collections.abc import Iterable, Sequence
52from dataclasses import dataclass, field
53from typing import Literal
54
55from sphinx_codelinks.analyse.models import SourceComment, WarningSubTypeEnum
56from sphinx_codelinks.config import CommentCategory, MultilineNeedsConfig
57from sphinx_codelinks.logger import get_logger
58
59logger = get_logger(__name__)
60
61CommentKind = Literal["line", "block", "docstring", "unknown"]
62
63#: line-comment delimiters, longest first so that ``///`` is not read as ``//``
64LINE_DELIMITERS: tuple[str, ...] = ("///", "//!", "//", "#")
65
66_LEADER = re.compile(r"^[ \t]*\*(?: |$)")
67"""A doxygen leader: optional whitespace, ``*``, then one space or the line end."""
68
69_OPENER_ONLY = re.compile(r"^/\*+!?\s*$")
70"""A block comment's first row holding nothing but its opener (``/*``, ``/*****``, ``/*!``)."""
71
72_CLOSER_ONLY = re.compile(r"^\s*\*+/\s*$")
73"""A block comment's last row holding nothing but its closer (``*/``, `` ***/``)."""
74
75_STARS = re.compile(r"^\s*\*+\s*")
76"""Stars at the start of a line, with the whitespace around them."""
77
78_STARS_ONLY = re.compile(r"^\s*\*+\s*$")
79"""A line of stars only: a separator, counted as a leader line and read as blank."""
80
81_DOCSTRING_OPEN = re.compile(r"""^([rRuUbBfF]{0,2})(\"\"\"|'''|"|')""")
82
83_HEADER = re.compile(r"^(?:\[([^\]]*)\])?[ \t]+([\w-]+):(?:[ \t]+(.*))?$")
84"""What follows the open word: ``[<markup>]``, whitespace, ``<type>:``, the title."""
85
86_OPTION = re.compile(r"^:([\w-]+):(?:[ \t]+(.*))?$")
87"""An option line, leading whitespace removed: ``:<key>:`` then the value."""
88
89RESERVED_KEYS: frozenset[str] = frozenset({"type", "title", "content", "doctype"})
90"""Keys the record sets itself: an option naming one duplicates it."""
91
92
93@dataclass(frozen=True)
94class LogicalLine:
95 """One line of a comment run with its comment prefix stripped."""
96
97 row: int
98 """The 0-based source row."""
99 col: int
100 """The 0-based source column (in characters) of ``text[0]``."""
101 text: str
102 """The line after prefix stripping."""
103
104
105@dataclass
106class CommentRun:
107 """The comments one block grammar pass reads, and their logical lines."""
108
109 comments: list[SourceComment]
110 lines: list[LogicalLine]
111 leaderless_block: bool = False
112 """A block comment whose lines did not all carry a ``*`` leader, so none was
113 stripped: an open word behind a star is reported, not silently lost."""
114 mixed_leaders: bool = False
115 """A leaderless block in which some lines DO carry the leader: the missing leader is
116 then the likely cause of an unterminated block (a plain block has none to miss)."""
117
118
119@dataclass
120class ParsedBlock:
121 """One multi-line need as the grammar found it, in source coordinates (0-based)."""
122
123 open_row: int
124 open_col: int
125 close_row: int
126 end_col: int
127 """One past the close word's last character."""
128 markup: str
129 doctype: str
130 need: dict[str, str]
131 option_rows: dict[str, int]
132 content_start: tuple[int, int] | None
133
134
135@dataclass(frozen=True)
136class ParseIssue:
137 """A refusal or a note of the grammar, at a 0-based source row."""
138
139 kind: WarningSubTypeEnum
140 row: int
141 msg: str
142
143
144@dataclass
145class ParseResult:
146 blocks: list[ParsedBlock] = field(default_factory=list)
147 issues: list[ParseIssue] = field(default_factory=list)
148 claimed_rows: set[int] = field(default_factory=set)
149
150
151@dataclass
152class _Comment:
153 comment: SourceComment
154 row: int
155 col: int
156 text: str
157 kind: CommentKind
158 delimiter: str
159 alone: bool
160 """Only whitespace before the comment on its row (or not knowable: libclang
161 without the row's text)."""
162
163
164def _text(comment: SourceComment) -> str:
165 raw = comment.node.text or b""
166 return raw.decode("utf-8", errors="replace")
167
168
169def _column(comment: SourceComment, text: str, row_text: str | None) -> int | None:
170 """The comment's 0-based column in characters, or ``None`` when not knowable.
171
172 A tree-sitter node gives a byte column; a libclang comment gives none, so its
173 column is read from the row: its first line is the row's suffix, or else its first
174 occurrence in the row (a one-row block comment followed by code).
175 """
176 node = comment.node
177 if getattr(node, "is_libclang", False):
178 if row_text is None:
179 return None
180 first = text.split("\n", 1)[0]
181 if row_text.endswith(first):
182 return len(row_text) - len(first)
183 index = row_text.find(first)
184 return index if index != -1 else None
185 byte_col = node.start_point.column
186 if row_text is None:
187 return byte_col
188 return len(row_text.encode("utf-8")[:byte_col].decode("utf-8", errors="replace"))
189
190
191def _classify(comment: SourceComment, text: str) -> tuple[CommentKind, str]:
192 # a libclang comment has no ``type``; only tree-sitter's Python grammar has docstrings
193 if getattr(comment.node, "type", None) == CommentCategory.docstring:
194 return "docstring", ""
195 stripped = text.lstrip()
196 if stripped.startswith("/*"):
197 return "block", "/*"
198 for delimiter in LINE_DELIMITERS:
199 if stripped.startswith(delimiter):
200 return "line", delimiter
201 return "unknown", ""
202
203
204def _view(comment: SourceComment, source_lines: Sequence[str] | None) -> _Comment:
205 text = _text(comment)
206 row = comment.node.start_point.row
207 row_text = (
208 source_lines[row]
209 if source_lines is not None and row < len(source_lines)
210 else None
211 )
212 col = _column(comment, text, row_text)
213 kind, delimiter = _classify(comment, text)
214 alone = True
215 if row_text is not None and col is not None:
216 alone = not row_text[:col].strip()
217 return _Comment(comment, row, col or 0, text, kind, delimiter, alone)
218
219
220def form_runs(
221 comments: Iterable[SourceComment], source_lines: Sequence[str] | None = None
222) -> list[CommentRun]:
223 """Group one file's comments into runs, in source order.
224
225 :param comments: The file's comment nodes, in any order (tree-sitter's captures are
226 not in source order).
227 :param source_lines: The file's rows, for what precedes a comment on its row; without
228 them every line comment counts as alone on its row.
229 """
230 views = sorted(
231 (_view(comment, source_lines) for comment in comments),
232 key=lambda view: (view.row, view.col),
233 )
234 runs: list[CommentRun] = []
235 current: list[_Comment] = []
236 for view in views:
237 if current and _continues(current[-1], view):
238 current.append(view)
239 continue
240 if current:
241 runs.append(_make_run(current))
242 current = [view]
243 if current:
244 runs.append(_make_run(current))
245 return runs
246
247
248def _continues(previous: _Comment, view: _Comment) -> bool:
249 return (
250 previous.kind == "line"
251 and view.kind == "line"
252 and previous.alone
253 and view.alone
254 and previous.delimiter == view.delimiter
255 and view.row == previous.row + 1
256 )
257
258
259def _make_run(views: list[_Comment]) -> CommentRun:
260 if len(views) == 1 and views[0].kind == "block":
261 lines, leader_stripped, mixed = _block_comment_lines(views[0])
262 return CommentRun([views[0].comment], lines, not leader_stripped, mixed)
263 run_lines: list[LogicalLine] = []
264 for view in views:
265 run_lines.extend(_logical_lines(view))
266 return CommentRun([view.comment for view in views], run_lines)
267
268
269def _logical_lines(view: _Comment) -> list[LogicalLine]:
270 if view.kind == "line":
271 return [_line_comment_line(view)]
272 if view.kind == "block":
273 return _block_comment_lines(view)[0]
274 if view.kind == "docstring":
275 return _docstring_lines(view)
276 logger.debug(
277 f"codelinks: a comment at row {view.row + 1} starts with no comment delimiter; "
278 "its lines are read unstripped"
279 )
280 return [
281 LogicalLine(view.row + offset, view.col if offset == 0 else 0, line)
282 for offset, line in enumerate(view.text.split("\n"))
283 ]
284
285
286def _line_comment_line(view: _Comment) -> LogicalLine:
287 text = view.text[:-1] if view.text.endswith("\n") else view.text
288 stripped = text.lstrip()
289 col = view.col + len(text) - len(stripped) + len(view.delimiter)
290 rest = stripped[len(view.delimiter) :]
291 if rest.startswith(" "):
292 rest = rest[1:]
293 col += 1
294 return LogicalLine(view.row, col, rest)
295
296
297def _block_comment_lines(view: _Comment) -> tuple[list[LogicalLine], bool, bool]:
298 """The block comment's logical lines, whether a ``*`` leader was stripped, and
299 whether, unstripped, some of its lines carry one (a mixed block)."""
300 texts = view.text.split("\n")
301 count = len(texts)
302 cols = [view.col] + [0] * (count - 1)
303 keep = [True] * count
304 if count > 1 and _OPENER_ONLY.match(texts[0].strip()):
305 keep[0] = False
306 else:
307 lead = len(texts[0]) - len(texts[0].lstrip())
308 body = texts[0].lstrip()
309 if body.startswith("/*!"):
310 opener = "/*!"
311 elif body.startswith("/**") and len(view.text.strip()) > 4:
312 opener = "/**"
313 else:
314 opener = "/*"
315 cols[0] += lead + len(opener)
316 texts[0] = body[len(opener) :]
317 if count > 1 and _CLOSER_ONLY.match(texts[-1]):
318 keep[-1] = False
319 else:
320 last = texts[-1].rstrip()
321 if last.endswith("*/"):
322 texts[-1] = last[: -len("*/")]
323
324 # the leader test reads the rows after the opener's, delimiter-only rows dropped
325 later = [i for i in range(1, count) if keep[i]]
326 non_blank = [i for i in later if texts[i].strip()]
327 leader_lines = [
328 i for i in non_blank if _LEADER.match(texts[i]) or _STARS_ONLY.match(texts[i])
329 ]
330 leader_stripped = bool(non_blank) and len(leader_lines) == len(non_blank)
331 mixed = not leader_stripped and bool(leader_lines)
332 if leader_stripped:
333 for i in later:
334 match = _LEADER.match(texts[i])
335 if match:
336 cols[i] += match.end()
337 texts[i] = texts[i][match.end() :]
338 elif _STARS_ONLY.match(texts[i]):
339 texts[i] = ""
340
341 if not texts[0].strip():
342 keep[0] = False
343 if count > 1 and not texts[-1].strip():
344 keep[-1] = False
345 lines = [
346 LogicalLine(view.row + i, cols[i], texts[i]) for i in range(count) if keep[i]
347 ]
348 return lines, leader_stripped, mixed
349
350
351def _docstring_lines(view: _Comment) -> list[LogicalLine]:
352 match = _DOCSTRING_OPEN.match(view.text)
353 if match is None:
354 return [LogicalLine(view.row, view.col, view.text)]
355 quote = match.group(2)
356 inner = view.text[match.end() :]
357 if inner.endswith(quote):
358 inner = inner[: -len(quote)]
359 raw = inner.split("\n")
360 first = raw[0]
361 lines = [
362 LogicalLine(
363 view.row,
364 view.col + match.end() + len(first) - len(first.lstrip()),
365 first.lstrip(),
366 )
367 ]
368 indents = [len(line) - len(line.lstrip()) for line in raw[1:] if line.strip()]
369 margin = min(indents, default=0)
370 for offset, line in enumerate(raw[1:], start=1):
371 text = line[margin:] if line.strip() else ""
372 lines.append(LogicalLine(view.row + offset, margin, text))
373 return lines
374
375
376def _indent(text: str) -> int:
377 return len(text) - len(text.lstrip())
378
379
380def open_rest(text: str, start_sequence: str) -> str | None:
381 """What follows the open word, if ``text`` starts with it as a whole word.
382
383 The word must be followed by ``[``, whitespace or the line end. So the default
384 reference marker ``@need-ids:``, which starts with the default open word, does not
385 open a block, and neither does ``@needle``.
386 """
387 stripped = text.lstrip()
388 if not stripped.startswith(start_sequence):
389 return None
390 rest = stripped[len(start_sequence) :]
391 if rest and rest[0] != "[" and not rest[0].isspace():
392 return None
393 return rest
394
395
[docs]396# @Parse multi-line need blocks in a comment run, IMPL_MLN_2, impl, [FE_MULTILINE_NEEDS]
397def parse_run(
398 lines: Sequence[LogicalLine],
399 config: MultilineNeedsConfig,
400 *,
401 leaderless_block: bool = False,
402 mixed_leaders: bool = False,
403) -> ParseResult:
404 """Find the multi-line needs in one comment run.
405
406 At each line opening a block: the close word ending the same line is the one-line
407 form, refused, and its line is claimed; no close line further down the run is an
408 unterminated block, refused, and the rest of the run is not scanned; an open line
409 that does not match the grammar is refused, and the lines up to its close are still
410 consumed. A block, a refused header and a refused one-line form claim their lines.
411
412 :param leaderless_block: The run is one block comment whose ``*`` leader was not
413 stripped: an open word behind a star is refused as a header, with the cause, and
414 consumed up to the first close line, which may then carry the stars too. The
415 refusal order is the same as for any open line: the one-line form first.
416 :param mixed_leaders: Some lines of that block do carry the leader: an unterminated
417 open then names the missing leader as the likely cause.
418 """
419 result = ParseResult()
420 start, end = config.start_sequence, config.end_sequence
421 unterminated = f"no '{end}' line before the comment ends; the block is skipped"
422 if leaderless_block and mixed_leaders:
423 unterminated += (
424 "; the likely cause: the block comment's lines do not all carry the '*' "
425 "leader"
426 )
427 index = 0
428 while index < len(lines):
429 line = lines[index]
430 rest = open_rest(line.text, start)
431 if rest is None:
432 stars = _STARS.match(line.text) if leaderless_block else None
433 starred_rest = open_rest(line.text[stars.end() :], start) if stars else None
434 if starred_rest is not None and _ends_with_word(starred_rest, end):
435 result.issues.append(_oneline_form(line.row, start, end))
436 result.claimed_rows.add(line.row)
437 index += 1
438 continue
439 if starred_rest is not None:
440 close = next(
441 (
442 k
443 for k in range(index + 1, len(lines))
444 if _is_close(lines[k].text, end, starred=True)
445 ),
446 None,
447 )
448 if close is None:
449 result.issues.append(
450 ParseIssue(
451 WarningSubTypeEnum.multiline_need_unterminated,
452 line.row,
453 unterminated,
454 )
455 )
456 break
457 result.issues.append(
458 ParseIssue(
459 WarningSubTypeEnum.multiline_need_header,
460 line.row,
461 f"'{start}' sits behind a '*' leader in a block comment "
462 "whose other lines carry none; give every line the leader "
463 "or none",
464 )
465 )
466 result.claimed_rows.update(
467 lines[k].row for k in range(index, close + 1)
468 )
469 index = close + 1
470 continue
471 index += 1
472 continue
473 if _ends_with_word(rest, end):
474 result.issues.append(_oneline_form(line.row, start, end))
475 result.claimed_rows.add(line.row)
476 index += 1
477 continue
478 close = next(
479 (
480 k
481 for k in range(index + 1, len(lines))
482 if _is_close(lines[k].text, end, starred=False)
483 ),
484 None,
485 )
486 if close is None:
487 result.issues.append(
488 ParseIssue(
489 WarningSubTypeEnum.multiline_need_unterminated,
490 line.row,
491 unterminated,
492 )
493 )
494 break
495 result.claimed_rows.update(lines[k].row for k in range(index, close + 1))
496 header = _HEADER.match(rest.rstrip())
497 if header is None:
498 result.issues.append(
499 ParseIssue(
500 WarningSubTypeEnum.multiline_need_header,
501 line.row,
502 f"the open line does not match '{start}[<markup>] <type>: <title>'; "
503 f"the block up to '{end}' is skipped",
504 )
505 )
506 else:
507 result.blocks.append(
508 _block(lines, index, close, header, config, result.issues)
509 )
510 index = close + 1
511 return result
512
513
514def _oneline_form(row: int, start: str, end: str) -> ParseIssue:
515 return ParseIssue(
516 WarningSubTypeEnum.multiline_need_oneline_form,
517 row,
518 f"'{start}' and '{end}' on one line: a one-line need is written with the "
519 "one-line marker; the block is skipped",
520 )
521
522
523def _is_close(text: str, end: str, *, starred: bool) -> bool:
524 """Whether ``text`` is a close line; ``starred`` also accepts stars before the word."""
525 stripped = text.strip()
526 if stripped == end:
527 return True
528 if starred:
529 stars = _STARS.match(stripped)
530 return stars is not None and stripped[stars.end() :] == end
531 return False
532
533
534def _ends_with_word(text: str, word: str) -> bool:
535 """Whether ``text`` ends with ``word`` preceded by whitespace or nothing."""
536 stripped = text.rstrip()
537 if not stripped.endswith(word):
538 return False
539 before = stripped[: -len(word)]
540 return not before or before[-1].isspace()
541
542
543def _block(
544 lines: Sequence[LogicalLine],
545 open_index: int,
546 close_index: int,
547 header: re.Match[str],
548 config: MultilineNeedsConfig,
549 issues: list[ParseIssue],
550) -> ParsedBlock:
551 open_line, close_line = lines[open_index], lines[close_index]
552 tag, need_type, title = header.group(1), header.group(2), header.group(3) or ""
553 markup = config.default_markup
554 if tag is not None:
555 if tag in config.markups:
556 markup = tag
557 else:
558 issues.append(
559 ParseIssue(
560 WarningSubTypeEnum.multiline_need_markup,
561 open_line.row,
562 f"unknown markup '{tag}' (known: "
563 f"{', '.join(sorted(config.markups))}); the project default "
564 f"'{config.default_markup}' is used",
565 )
566 )
567
568 options: dict[str, str] = {}
569 option_rows: dict[str, int] = {}
570 index = open_index + 1
571 # the option a continuation line extends: its key (None when the option was
572 # refused as a duplicate) and its indentation
573 current: tuple[str | None, int] | None = None
574 while index < close_index:
575 text = lines[index].text
576 if current is not None and text.strip() and _indent(text) > current[1]:
577 key = current[0]
578 if key is not None:
579 options[key] = " ".join(filter(None, (options[key], text.strip())))
580 index += 1
581 continue
582 option = _OPTION.match(text.strip())
583 if option is None:
584 break
585 key, value = option.group(1), (option.group(2) or "").strip()
586 if key in options or key in RESERVED_KEYS:
587 what = "the block's own" if key in RESERVED_KEYS else "an earlier"
588 issues.append(
589 ParseIssue(
590 WarningSubTypeEnum.multiline_need_duplicate_option,
591 lines[index].row,
592 f"option '{key}' repeats {what} '{key}'; the first value is kept",
593 )
594 )
595 current = (None, _indent(text))
596 else:
597 options[key] = value
598 option_rows[key] = lines[index].row + 1
599 current = (key, _indent(text))
600 index += 1
601
602 body = list(lines[index:close_index])
603 while body and not body[0].text.strip():
604 body.pop(0)
605 while body and not body[-1].text.strip():
606 body.pop()
607 content_lines: list[str] = []
608 content_start: tuple[int, int] | None = None
609 if body:
610 content_start = (body[0].row, body[0].col + _indent(body[0].text))
611 margin = min(_indent(line.text) for line in body if line.text.strip())
612 for line in body:
613 text = line.text[margin:] if line.text.strip() else ""
614 stripped = text.lstrip()
615 if stripped.startswith("\\" + config.end_sequence):
616 # tested past the indentation, as the close is; the indentation stays
617 text = text[: len(text) - len(stripped)] + stripped[1:]
618 elif open_rest(text, config.start_sequence) is not None:
619 issues.append(
620 ParseIssue(
621 WarningSubTypeEnum.multiline_need_nested_open,
622 line.row,
623 f"'{config.start_sequence}' inside a multi-line need's body is "
624 "body text: one need per block",
625 )
626 )
627 content_lines.append(text)
628
629 need: dict[str, str] = {"type": need_type, "title": title.strip()}
630 need.update(options)
631 need["content"] = "\n".join(content_lines)
632 need["doctype"] = config.markups[markup]
633 return ParsedBlock(
634 open_row=open_line.row,
635 open_col=open_line.col + _indent(open_line.text),
636 close_row=close_line.row,
637 end_col=close_line.col
638 + close_line.text.index(config.end_sequence)
639 + len(config.end_sequence),
640 markup=markup,
641 doctype=need["doctype"],
642 need=need,
643 option_rows=option_rows,
644 content_start=content_start,
645 )
646
647
648def blank_rows(text: str, first_row: int, rows: set[int]) -> str:
649 """``text`` with each of ``rows`` (0-based source rows) replaced by an empty line.
650
651 The text keeps its line count, so the row arithmetic of the other extractors holds.
652 """
653 lines = text.split("\n")
654 return "\n".join(
655 "" if first_row + offset in rows else line for offset, line in enumerate(lines)
656 )
Back to top