1from collections import deque
2from dataclasses import MISSING, dataclass, field, fields, replace
3from enum import Enum
4from pathlib import Path
5from typing import Any, Literal, TypedDict, cast
6
7from jsonschema import ValidationError, validate
8from sphinx.application import Sphinx
9from sphinx.config import Config as _SphinxConfig
10
11from sphinx_codelinks.source_discover.config import (
12 CommentType,
13 SourceDiscoverConfig,
14 SourceDiscoverSectionConfigType,
15)
16from sphinx_codelinks.source_discover.source_discover import SourceDiscover
17from ub_project import anchor, load_toml, select_table
18
19UNIX_NEWLINE = "\n"
20
21
22COMMENT_MARKERS = {
[docs] 23 # @Support C and C++ style comments, IMPL_C_1, impl, [FE_C_SUPPORT, FE_CPP]
24 CommentType.cpp: ["//", "/*"],
[docs] 25 # @Support Python style comments, IMPL_PY_1, impl, [FE_PY]
26 CommentType.python: ["#"],
27 CommentType.cs: ["//", "/*", "///"],
[docs] 28 # @Support Go style comments, IMPL_GO_2, impl, [FE_GO]
29 CommentType.go: ["//", "/*"],
30}
31ESCAPE = "\\"
32
33# Default C/C++ standard for the standalone/defines parse path when no
34# compile_commands.json entry supplies one.
35DEFAULT_CPP_STD = "c++17"
36
37
38class CommentCategory(str, Enum): # noqa: UP042 # StrEnum changes str(member), which reaches CLI warnings and error messages
39 comment = "comment"
40 docstring = "expression_statement"
41
42
43class NeedIdRefsConfigType(TypedDict):
44 markers: list[str]
45
46
47@dataclass
48class NeedIdRefsConfig:
49 @classmethod
50 def field_names(cls) -> set[str]:
51 return {item.name for item in fields(cls)}
52
53 markers: list[str] = field(
54 default_factory=lambda: ["@need-ids:"],
55 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
56 )
57 """The markers to extract need ids from"""
58
59 @classmethod
60 def get_schema(cls, name: str) -> dict[str, Any] | None:
61 _field = next(_field for _field in fields(cls) if _field.name is name)
62 if _field.metadata and "schema" in _field.metadata:
63 return cast(dict[str, Any], _field.metadata["schema"])
64 return None
65
66 def check_schema(self) -> list[str]:
67 errors = []
68 for _field_name in self.field_names():
69 schema = self.get_schema(_field_name)
70 value = getattr(self, _field_name)
71 try:
72 # jsonschema's stubs type `schema` as `bool | Mapping`, but
73 # `get_schema` returns an optional dict
74 validate(
75 instance=value,
76 schema=schema, # ty: ignore[invalid-argument-type]
77 )
78 except ValidationError as e:
79 errors.append(
80 f"Schema validation error in field '{_field_name}': {e.message}"
81 )
82 return errors
83
84
85class MultilineNeedsConfigType(TypedDict, total=False):
86 start_sequence: str
87 end_sequence: str
88 default_markup: str
89 markups: dict[str, str]
90
91
92DEFAULT_MARKUPS: dict[str, str] = {"rst": ".rst", "md": ".md"}
93"""The default ``markups`` table: markup tag -> the ``doctype`` suffix of the need."""
94
95
96@dataclass
97class MultilineNeedsConfig:
98 """The markers and markups of multi-line needs (``[analyse.multiline_needs]``)."""
99
100 @classmethod
101 def field_names(cls) -> set[str]:
102 return {item.name for item in fields(cls)}
103
104 start_sequence: str = field(
105 default="@need", metadata={"schema": {"type": "string", "minLength": 1}}
106 )
107 """The word that opens a multi-line need, at the start of its line."""
108 end_sequence: str = field(
109 default="@endneed", metadata={"schema": {"type": "string", "minLength": 1}}
110 )
111 """The word that closes a multi-line need, alone on its line."""
112 default_markup: str = field(
113 default="rst", metadata={"schema": {"type": "string", "minLength": 1}}
114 )
115 """The markup tag of a block whose open line names none; a key of ``markups``."""
116 markups: dict[str, str] = field(
117 default_factory=lambda: dict(DEFAULT_MARKUPS),
118 metadata={
119 "schema": {
120 "type": "object",
121 "additionalProperties": {"type": "string"},
122 "minProperties": 1,
123 }
124 },
125 )
126 """Markup tag -> the ``doctype`` suffix the need's content is parsed with. Not
127 checked against any parser: the consumer decides what it can parse."""
128
129 @classmethod
130 def get_schema(cls, name: str) -> dict[str, Any] | None:
131 _field = next(_field for _field in fields(cls) if _field.name is name)
132 if _field.metadata and "schema" in _field.metadata:
133 return cast(dict[str, Any], _field.metadata["schema"])
134 return None
135
136 def check_schema(self) -> list[str]:
137 errors = []
138 for _field_name in self.field_names():
139 schema = self.get_schema(_field_name)
140 value = getattr(self, _field_name)
141 try:
142 # jsonschema's stubs type `schema` as `bool | Mapping`, but
143 # `get_schema` returns an optional dict
144 validate(
145 instance=value,
146 schema=schema, # ty: ignore[invalid-argument-type]
147 )
148 except ValidationError as e:
149 errors.append(
150 f"Schema validation error in field '{_field_name}': {e.message}"
151 )
152 return errors
153
154 def check_fields_configuration(self) -> list[str]:
155 errors = self.check_schema()
156 if self.start_sequence == self.end_sequence:
157 errors.append("start_sequence and end_sequence cannot be the same.")
158 if isinstance(self.markups, dict) and self.default_markup not in self.markups:
159 errors.append(
160 f"default_markup {self.default_markup!r} is not a key of markups "
161 f"({', '.join(repr(tag) for tag in sorted(self.markups))})."
162 )
163 return errors
164
165
166@dataclass
167class PreprocessorConfig:
168 """Opt-in libclang engine config. Presence => libclang engine for C/C++."""
169
170 compile_commands: Path | None = field(
171 default=None, metadata={"schema": {"type": ["string", "null"]}}
172 )
173 """Explicit path to compile_commands.json. If None, walk-up auto-discovery."""
174
175 defines: list[str] = field(
176 default_factory=list,
177 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
178 )
179 """Fallback -D defines applied globally when no compile_commands.json applies."""
180
181 includes: list[Path] = field(
182 default_factory=list,
183 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
184 )
185 """Fallback -I include dirs for the defines path."""
186
187 std: str = field(
188 default=DEFAULT_CPP_STD,
189 metadata={"schema": {"type": "string"}},
190 )
191 """C/C++ standard for the standalone/defines parse path (e.g. ``c++17``,
192 ``c++20``, ``c11``); libclang pins ``-x`` to match it. Files resolved from a
193 ``compile_commands.json`` entry use that entry's own ``-std`` instead."""
194
195
196def anchor_preproc_paths(preproc: PreprocessorConfig, base: Path) -> PreprocessorConfig:
197 """Resolve a preprocessor config's ``compile_commands`` and ``includes``
198 against ``base`` (the config file's directory), so a relative path resolves
199 against the TOML file rather than the process CWD — matching ``src_dir`` /
200 ``git_root``. An absolute path is not anchored, but it is resolved too
201 (symlinks followed, ``..`` folded), like every other path here.
202 """
203 return replace(
204 preproc,
205 compile_commands=(
206 anchor(preproc.compile_commands, base).resolve()
207 if preproc.compile_commands is not None
208 else None
209 ),
210 includes=[anchor(inc, base).resolve() for inc in preproc.includes],
211 )
212
213
214class FieldConfig(TypedDict, total=False):
215 name: str
216 type: Literal["str", "list[str]"]
217 default: str | list[str] | None
218
219
220class OneLineCommentStyleType(TypedDict):
221 start_sequence: str
222 end_sequence: str
223 field_split_char: str
224 needs_fields: list[FieldConfig]
225
226
227@dataclass
228class OneLineCommentStyle:
229 def __setattr__(self, name: str, value: Any) -> None:
230 if name == "needs_fields":
231 # apply default to fields
232 self.apply_needs_field_default(value)
233 return super().__setattr__(name, value)
234
235 @classmethod
236 def field_names(cls) -> set[str]:
237 return {item.name for item in fields(cls)}
238
239 start_sequence: str = field(default="@", metadata={"schema": {"type": "string"}})
240 """Chars sequence to indicate the start of the one-line comment."""
241
242 end_sequence: str = field(
243 default=UNIX_NEWLINE, metadata={"schema": {"type": "string"}}
244 )
245 """Chars sequence to indicate the end of the one-line comment."""
246
247 field_split_char: str = field(default=",", metadata={"schema": {"type": "string"}})
248 """Char sequence to split the fields."""
249
250 needs_fields: list[FieldConfig] = field(
251 default_factory=lambda: [
252 {"name": "title"},
253 {"name": "id"},
254 {"name": "type", "default": "impl"},
255 {"name": "links", "type": "list[str]", "default": []},
256 ],
257 metadata={
258 "required_fields": ["title", "type"],
259 "field_default": {
260 "type": "str",
261 },
262 "schema": {
263 "type": "array",
264 "items": {
265 "type": "object",
266 "properties": {
267 "name": {"type": "string"},
268 "type": {
269 "type": "string",
270 "enum": ["str", "list[str]"],
271 "default": "str",
272 },
273 "default": {
274 "anyOf": [
275 {"type": "string"},
276 {"type": "array", "items": {"type": "string"}},
277 ]
278 },
279 },
280 "required": ["name"],
281 "additionalProperties": False,
282 "allOf": [
283 {
284 "if": {"properties": {"type": {"const": "list[str]"}}},
285 "then": {
286 "properties": {
287 "default": {
288 "type": "array",
289 "items": {"type": "string"},
290 }
291 }
292 },
293 },
294 {
295 "if": {"properties": {"type": {"const": "str"}}},
296 "then": {"properties": {"default": {"type": "string"}}},
297 },
298 ],
299 },
300 },
301 },
302 )
303
304 @classmethod
305 def apply_needs_field_default(cls, given_fields: list[FieldConfig]) -> None:
306 field_default = next(
307 _field.metadata["field_default"]
308 for _field in fields(cls)
309 if _field.name == "needs_fields"
310 )
311
312 for _field in given_fields:
313 for _default in field_default:
314 if _default not in _field:
315 _field[_default] = field_default[
316 _default
317 ] # dynamically assign keys
318
319 @classmethod
320 def get_required_fields(cls, name: str) -> list[str] | None:
321 _field = next(_field for _field in fields(cls) if _field.name is name)
322 if _field.metadata:
323 return cast(list[str], _field.metadata["required_fields"])
324 return None
325
326 @classmethod
327 def get_schema(cls, name: str) -> dict[str, Any] | None:
328 _field = next(_field for _field in fields(cls) if _field.name is name)
329 if _field.metadata and "schema" in _field.metadata:
330 return cast(dict[str, Any], _field.metadata["schema"])
331 return None
332
333 def check_schema(self) -> list[str]:
334 errors = []
335 for _field_name in self.field_names():
336 schema = self.get_schema(_field_name)
337 value = getattr(self, _field_name)
338 try:
339 validate(
340 instance=value,
341 schema=schema, # ty: ignore[invalid-argument-type]
342 ) # validate has no type specified
343 except ValidationError as e:
344 if _field_name == "needs_fields":
345 need_field_name = value[e.path[0]]["name"]
346 errors.append(
347 f"Schema validation error in need_fields '{need_field_name}': {e.message}"
348 )
349 else:
350 errors.append(
351 f"Schema validation error in field '{_field_name}': {e.message}"
352 )
353 return errors
354
355 def check_required_fields(self) -> list[str]:
356 errors = []
357 required_fields = self.get_required_fields("needs_fields")
358 if required_fields is None:
359 errors.append("No required fields specified.")
360 return errors
361 given_field_names = [_field["name"] for _field in self.needs_fields]
362 missing_fields = set(required_fields) - set(given_field_names)
363 if len(missing_fields) != 0:
364 errors.append(f"Missing required fields: {sorted(missing_fields)}")
365
366 return errors
367
368 def check_fields_mutually_exclusive(self) -> list[str]:
369 errors = []
370 needs_field_names = set()
371 for _field in self.needs_fields:
372 if _field["name"] in needs_field_names:
373 errors.append(f"Field '{_field['name']}' is defined multiple times.")
374 needs_field_names.add(_field["name"])
375 return errors
376
377 def check_fields_default_order(self) -> list[str]:
378 errors = []
379 seen_default = False
380 first_default_field = ""
381 for _field in self.needs_fields:
382 has_default = _field.get("default") is not None
383 if has_default and not seen_default:
384 seen_default = True
385 first_default_field = _field["name"]
386 elif not has_default and seen_default:
387 errors.append(
388 f"Field '{_field['name']}' without a default follows "
389 f"field '{first_default_field}' which has a default. "
390 f"Fields without defaults must be defined before fields with defaults."
391 )
392 return errors
393
394 def check_fields_configuration(self) -> list[str]:
395 return (
396 self.check_schema()
397 + self.check_required_fields()
398 + self.check_fields_mutually_exclusive()
399 + self.check_fields_default_order()
400 )
401
402 def get_cnt_required_fields(self) -> int:
403 cnt_required_fields = 0
404 for _field in self.needs_fields:
405 if _field.get("default") is None:
406 cnt_required_fields += 1
407 return cnt_required_fields
408
409 def get_pos_list_str(self) -> list[int]:
410 pos_list_str = []
411 for idx, _field in enumerate(self.needs_fields):
412 if _field["type"] == "list[str]":
413 pos_list_str.append(idx + 1)
414 return pos_list_str
415
416
417class AnalyseSectionConfigType(TypedDict, total=False):
418 """Define typing for loading `analyse` section from the file."""
419
420 get_need_id_refs: bool
421 get_oneline_needs: bool
422 get_multiline_needs: bool
423 outdir: str
424 git_root: str
425 need_id_refs: NeedIdRefsConfigType
426 multiline_needs: MultilineNeedsConfigType
427 oneline_comment_style: OneLineCommentStyleType
428 preprocessor: dict[str, object]
429
430
431class SourceAnalyseConfigType(TypedDict, total=False):
432 """Define typing for its API configuration."""
433
434 src_files: list[Path]
435 src_dir: Path
436 comment_type: CommentType
437 get_need_id_refs: bool
438 get_oneline_needs: bool
439 get_multiline_needs: bool
440 git_root: Path | None
441 need_id_refs_config: NeedIdRefsConfig
442 multiline_needs_config: MultilineNeedsConfig
443 oneline_comment_style: OneLineCommentStyle
444 preprocessor: PreprocessorConfig | None
445
446
447class ProjectsAnalyseConfigType(TypedDict, total=False):
448 projects_config: dict[str, SourceAnalyseConfigType]
449
450
451@dataclass
452class SourceAnalyseConfig:
453 @classmethod
454 def field_names(cls) -> set[str]:
455 return {item.name for item in fields(cls)}
456
457 src_files: list[Path] = field(
458 default_factory=list,
459 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
460 )
461 """A list of source files to be processed."""
462 src_dir: Path = field(
463 default_factory=lambda: Path("./"), metadata={"schema": {"type": "string"}}
464 )
465
466 comment_type: CommentType = field(
467 default=CommentType.cpp, metadata={"schema": {"type": "string"}}
468 )
469 """The type of comment to be processed."""
470
471 get_need_id_refs: bool = field(
472 default=True, metadata={"schema": {"type": "boolean"}}
473 )
474 """Whether to extract need id references from comments"""
475
476 get_oneline_needs: bool = field(
477 default=False, metadata={"schema": {"type": "boolean"}}
478 )
479 """Whether to extract oneline needs from comments"""
480
481 get_multiline_needs: bool = field(
482 default=False, metadata={"schema": {"type": "boolean"}}
483 )
484 """Whether to extract multi-line needs (``@need`` … ``@endneed``) from comments"""
485
486 git_root: Path | None = field(
487 default=None, metadata={"schema": {"type": ["string", "null"]}}
488 )
489 """Explicit path to the Git repository root. If not set, it will be auto-detected
490 by traversing parent directories. Useful for Bazel builds or deeply nested configs."""
491
492 need_id_refs_config: NeedIdRefsConfig = field(default_factory=NeedIdRefsConfig)
493 """Configuration for extracting need id references from comments."""
494
495 multiline_needs_config: MultilineNeedsConfig = field(
496 default_factory=MultilineNeedsConfig
497 )
498 """Configuration for extracting multi-line needs from comments."""
499
500 oneline_comment_style: OneLineCommentStyle = field(
501 default_factory=OneLineCommentStyle
502 )
503 """Configuration for extracting oneline needs from comments."""
504
505 preprocessor: PreprocessorConfig | None = field(default=None)
506 """Opt-in libclang preprocessor engine. None => tree-sitter (default).
507
508 No flat ``metadata["schema"]`` here: this is a nested dataclass, like the
509 sibling ``need_id_refs_config`` / ``multiline_needs_config`` /
510 ``oneline_comment_style`` fields. ``check_schema`` only validates fields that
511 declare a flat schema; giving this field one made it validate the constructed
512 ``PreprocessorConfig`` instance against JSON type ``object`` and fail at
513 ``config-inited``. Its structure is enforced by ``convert_analyse_config``.
514 """
515
516 @classmethod
517 def get_schema(cls, name: str) -> dict[str, Any] | None:
518 _field = next(_field for _field in fields(cls) if _field.name is name)
519 if _field.metadata and "schema" in _field.metadata:
520 return cast(dict[str, Any], _field.metadata["schema"])
521 return None
522
523 def check_schema(self) -> list[str]:
524 errors = []
525 for _field_name in self.field_names():
526 schema = self.get_schema(_field_name)
527 if not schema:
528 continue
529 value = getattr(self, _field_name)
530 if isinstance(value, Path): # adapt to json schema restriction
531 value = str(value)
532 if _field_name == "src_files" and isinstance(
533 value, list
534 ): # adapt to json schema restriction
535 value: list[str] = [
536 str(src_file) for src_file in value
537 ] # only for value adaptation
538 try:
539 validate(instance=value, schema=schema)
540 except ValidationError as e:
541 errors.append(
542 f"Schema validation error in field '{_field_name}': {e.message}"
543 )
544 return errors
545
546 def check_markers_mutually_exclusive(self) -> list[str]:
547 errors = set()
548 markers = set()
549 markers.add(self.oneline_comment_style.start_sequence)
550 markers.add(self.oneline_comment_style.end_sequence)
551 # equality only: the default one-line start ``@`` is a PREFIX of ``@need``, and
552 # what keeps the two apart is that a block's lines are hidden from the one-line
553 # parser, not this check. The two words are checked only when the feature is
554 # on: off, they are no markers of the project.
555 multiline_markers = (
556 (
557 self.multiline_needs_config.start_sequence,
558 self.multiline_needs_config.end_sequence,
559 )
560 if self.get_multiline_needs
561 else ()
562 )
563 for marker in multiline_markers:
564 if marker in markers:
565 errors.add(f"Marker {marker} is defined multiple times")
566 else:
567 markers.add(marker)
568
569 for marker in self.need_id_refs_config.markers:
570 if marker in markers:
571 errors.add(f"Marker {marker} is defined multiple times")
572 else:
573 markers.add(marker)
574 return list(errors)
575
576 def check_fields_configuration(self) -> list[str]:
577 errors: deque[str] = deque()
578 if self.get_need_id_refs:
579 need_id_refs_errors = self.need_id_refs_config.check_schema()
580 if need_id_refs_errors:
581 errors.appendleft("NeedIdRefs configuration errors:")
582 errors.extend(need_id_refs_errors)
583 if self.get_oneline_needs:
584 oneline_needs_errors = (
585 self.oneline_comment_style.check_fields_configuration()
586 )
587 if oneline_needs_errors:
588 errors.appendleft("OneLineCommentStyle configuration errors:")
589 errors.extend(oneline_needs_errors)
590 if self.get_multiline_needs:
591 multiline_errors = self.multiline_needs_config.check_fields_configuration()
592 if multiline_errors:
593 errors.appendleft("MultilineNeeds configuration errors:")
594 errors.extend(multiline_errors)
595 analyse_errors = self.check_markers_mutually_exclusive() + self.check_schema()
596 if analyse_errors:
597 errors.appendleft("analyse configuration errors:")
598 errors.extend(analyse_errors)
599 return list(errors)
600
601
602# Shared ubCode project file, which other useblocks tools (sphinx-needs,
603# ubCode checker, ...) read as well, so all tools see the same projects.
604DEFAULT_CONFIG_TOML: str = "ubproject.toml"
605
606DEFAULT_REF_URL_FIELD: str = "code_url"
607"""The need field ``@need-ids:`` references are attached to, unless a project's
608``ref_url_field`` names another (ubCode's key and default; ``""`` disables the attach)."""
609
610#: The table of the TOML file that both readers, the Sphinx extension and the CLI,
611#: take their configuration from.
612CODELINKS_TABLE: str = "codelinks"
613
614
615def load_codelinks_table(path: Path) -> dict[str, object] | None:
616 """Parse *path* and return its ``[codelinks]`` table, through ub-project.
617
618 The values are returned RAW: relative paths are anchored where they are used
619 (the Sphinx extension anchors at ``confdir / Path(config_from_toml).parent``,
620 unresolved), never here.
621
622 :param path: The TOML file.
623 :return: The table, or ``None`` when the file has no ``codelinks`` key.
624 :raises ub_project.ProjectConfigError: If the file cannot be read, is not UTF-8
625 or not valid TOML, or if ``codelinks`` is not a table.
626 :raises RecursionError: For a pathologically nested file, which ``load_toml``
627 does not wrap -- why both callers also catch ``Exception``.
628 """
629 return select_table(load_toml(path), CODELINKS_TABLE, source=path)
630
631
632class CodeLinksProjectConfigType(TypedDict, total=False):
633 """TypedDict defining the configuration structure for individual SrcTrace projects.
634
635 Contains both user-provided configuration:
636 - source_discover
637 - remote_url_pattern
638 - ref_url_field
639 - analyse
640 and runtime-generated configuration objects
641 - source_discover_config
642 - analyse_config
643 """
644
645 source_discover: SourceDiscoverSectionConfigType
646 remote_url_pattern: str
647 ref_url_field: str
648 analyse: AnalyseSectionConfigType
649 source_discover_config: SourceDiscoverConfig
650 analyse_config: SourceAnalyseConfig
651
652
653class CodeLinksConfigType(TypedDict):
654 config_from_toml: str | None
655 set_local_url: bool
656 local_url_field: str
657 set_remote_url: bool
658 remote_url_field: str
659 outdir: Path
660 projects: dict[str, CodeLinksProjectConfigType]
661 debug_measurement: bool
662 debug_filters: bool
663
664
665@dataclass
666class CodeLinksConfig:
667 @classmethod
668 def from_sphinx(cls, sphinx_config: _SphinxConfig) -> "CodeLinksConfig":
669 obj = cls()
670 src_trace_projects = getattr(sphinx_config, "src_trace_projects", None)
671 if isinstance(src_trace_projects, dict):
672 generate_project_configs(src_trace_projects)
673 super().__setattr__(obj, "_sphinx_config", sphinx_config)
674 return obj
675
676 def __getattribute__(self, name: str) -> Any:
677 if name.startswith("__") or name == "_sphinx_config":
678 return super().__getattribute__(name)
679 sphinx_config = (
680 object.__getattribute__(self, "_sphinx_config")
681 if "_sphinx_config" in self.__dict__
682 else None
683 )
684 if sphinx_config:
685 return getattr(
686 super().__getattribute__("_sphinx_config"), f"src_trace_{name}"
687 )
688
689 return object.__getattribute__(self, name)
690
691 def __setattr__(self, name: str, value: Any) -> None:
692 if name == "_sphinx_config" and "src_trace_projects" in value:
693 src_trace_projects: dict[str, CodeLinksProjectConfigType] = value[
694 "src_trace_projects"
695 ]
696 generate_project_configs(src_trace_projects)
697
698 if name.startswith("__") or name == "_sphinx_config":
699 return super().__setattr__(name, value)
700
701 sphinx_config = (
702 object.__getattribute__(self, "_sphinx_config")
703 if "_sphinx_config" in self.__dict__
704 else None
705 )
706
707 if sphinx_config:
708 setattr(
709 super().__getattribute__("_sphinx_config"), f"src_trace_{name}", value
710 )
711
712 if name == "outdir" and isinstance(value, str):
713 # Ensure outdir is a Path object
714 value = Path(value)
715 return object.__setattr__(self, name, value)
716
717 @classmethod
718 def add_config_values(cls, app: Sphinx) -> None:
719 """Add all config values to Sphinx application"""
720 for item in fields(cls):
721 if item.default_factory is not MISSING:
722 default = item.default_factory()
723 elif item.default is not MISSING:
724 default = item.default
725 else:
726 raise Exception(f"Field {item.name} has no default value or factory")
727
728 name = item.name
729 app.add_config_value(
730 f"src_trace_{name}",
731 default,
732 item.metadata["rebuild"],
733 types=item.metadata["types"],
734 )
735
736 @classmethod
737 def field_names(cls) -> set[str]:
738 return {item.name for item in fields(cls)}
739
740 @classmethod
741 def get_schema(cls, name: str) -> dict[str, Any] | None:
742 """Get the schema for a config item."""
743 _field = next(field for field in fields(cls) if field.name is name)
744 if _field.metadata and "schema" in _field.metadata:
745 return _field.metadata["schema"]
746 return None
747
748 config_from_toml: str | None = field(
749 default=DEFAULT_CONFIG_TOML,
750 metadata={
751 "rebuild": "env",
752 "types": (str, type(None)),
753 "schema": {
754 "type": ["string", "null"],
755 "examples": [DEFAULT_CONFIG_TOML, None],
756 },
757 },
758 )
759 """Path to a TOML file to load configuration from.
760
761 Defaults to ``ubproject.toml`` next to :file:`conf.py`. The default is the
762 value ``ubproject.toml`` exactly -- left unset, or written in conf.py as that
763 string: missing, or without a ``[codelinks]`` table, it is silently ignored.
764 Any other value, ``./ubproject.toml`` included, is an explicit file and warns
765 in both cases. A file that exists but cannot be read or parsed warns
766 (``codelinks.config``) either way, as any configured file did at 1.4.0.
767 """
768
769 set_local_url: bool = field(
770 default=False,
771 metadata={
772 "rebuild": "env",
773 "types": (bool,),
774 "schema": {
775 "type": "boolean",
776 },
777 },
778 )
779 """Set the file URL in the extracted need."""
780
781 local_url_field: str = field(
782 default="local-url",
783 metadata={
784 "rebuild": "env",
785 "types": (str,),
786 "schema": {
787 "type": "string",
788 },
789 },
790 )
791 """The field name for the file URL in the extracted need."""
792
793 set_remote_url: bool = field(
794 default=False,
795 metadata={
796 "rebuild": "env",
797 "types": (bool,),
798 "schema": {
799 "type": "boolean",
800 },
801 },
802 )
803 remote_url_field: str = field(
804 default="remote-url",
805 metadata={
806 "rebuild": "env",
807 "types": (str,),
808 "schema": {
809 "type": "string",
810 },
811 },
812 )
813 """The field name for the remote URL in the extracted need."""
814
815 outdir: Path = field(
816 default=Path("output"),
817 metadata={"rebuild": "env", "types": (str), "schema": {"type": "string"}},
818 )
819 """The directory where the generated artifacts and their caches will be stored."""
820
821 projects: dict[str, CodeLinksProjectConfigType] = field(
822 default_factory=dict,
823 metadata={
824 "rebuild": "env",
825 "types": (),
826 "schema": {
827 "type": "object",
828 "additionalProperties": {
829 "type": "object",
830 "properties": {
831 "source_discover": {},
832 "analyse": {},
833 "remote_url_pattern": {},
834 "ref_url_field": {},
835 "source_discover_config": {},
836 "analyse_config": {},
837 },
838 "additionalProperties": False,
839 },
840 },
841 },
842 )
843 """The configuration for the source tracing projects."""
844
845 debug_measurement: bool = field(
846 default=False, metadata={"rebuild": "html", "types": (bool,)}
847 )
848 """If True, log runtime information for various functions."""
849 debug_filters: bool = field(
850 default=False, metadata={"rebuild": "html", "types": (bool,)}
851 )
852 """If True, log filter processing runtime information."""
853
854
855def check_schema(config: CodeLinksConfig) -> list[str]:
856 """Check only first layer's of schema, so that the nested dict is not validated here."""
857 errors = []
858 for _field_name in CodeLinksConfig.field_names():
859 schema = CodeLinksConfig.get_schema(_field_name)
860 if not schema:
861 continue
862 value = getattr(config, _field_name)
863 if isinstance(value, Path): # adapt to json schema restriction
864 value = str(value)
865 try:
866 validate(instance=value, schema=schema)
867 except ValidationError as e:
868 errors.append(
869 f"Schema validation error in filed '{_field_name}': {e.message}"
870 )
871 return errors
872
873
874def check_project_configuration(config: CodeLinksConfig) -> list[str]:
875 """Check nested project configurations"""
876 errors = []
877
878 for project_name, project_config in config.projects.items():
879 project_errors: list[str] = []
880
881 # validate source_discover config
882 src_discover_config: SourceDiscoverConfig | None = project_config.get(
883 "source_discover_config"
884 )
885 src_discover_errors = []
886 if src_discover_config:
887 src_discover_errors.extend(src_discover_config.check_schema())
888
889 # validate analyse config
890 analyse_config: SourceAnalyseConfig | None = project_config.get(
891 "analyse_config"
892 )
893 analyse_errors = []
894 if analyse_config:
895 analyse_errors = analyse_config.check_fields_configuration()
896
897 # validate src-trace config
898 if config.set_remote_url and "remote_url_pattern" not in project_config:
899 project_errors.append(
900 "remote_url_pattern must be given, as set_remote_url is enabled"
901 )
902
903 if "remote_url_pattern" in project_config and not isinstance(
904 project_config["remote_url_pattern"], str
905 ):
906 project_errors.append("remote_url_pattern must be a string")
907
908 ref_url_field = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
909 if not isinstance(ref_url_field, str):
910 project_errors.append("ref_url_field must be a string")
911 elif ref_url_field and ref_url_field in _url_fields(config):
912 project_errors.append(
913 f"ref_url_field {ref_url_field!r} must differ from local_url_field "
914 "and remote_url_field"
915 )
916
917 if analyse_errors or src_discover_errors or project_errors:
918 errors.append(f"Project '{project_name}' has the following errors:")
919 errors.extend(analyse_errors)
920 errors.extend(src_discover_errors)
921 errors.extend(project_errors)
922
923 return errors
924
925
926def config_base_dir(confdir: str | Path, config: CodeLinksConfig) -> Path:
927 """The directory a project's relative paths are anchored at.
928
929 The configuration file's directory when ``src_trace_config_from_toml`` names one
930 (itself anchored at ``confdir``), else ``confdir``: ``src_dir``, ``git_root`` and
931 the preprocessor paths all resolve against it.
932 """
933 base = Path(confdir)
934 if config.config_from_toml:
935 base = anchor(Path(config.config_from_toml).parent, base)
936 return base
937
938
939def locate_src_dir(
940 confdir: str | Path,
941 config: CodeLinksConfig,
942 discover_config: SourceDiscoverConfig,
943) -> Path:
944 """A project's source directory, anchored (see :func:`config_base_dir`) and resolved."""
945 return anchor(discover_config.src_dir, config_base_dir(confdir, config)).resolve()
946
947
948def git_root_problem(git_root: Path, src_dir: Path) -> str | None:
949 """Why a configured ``git_root`` cannot be a project's git root, or ``None``.
950
951 Every source path is relative to the git root, so it must be ``src_dir`` or a
952 directory above it (#2062). Both paths anchored and resolved by the caller: the
953 Sphinx extension (``project_analysis.configured_git_root``) and ``codelinks
954 analyse`` alike, each treating a value with a problem as unset.
955 """
956 try:
957 if not git_root.exists():
958 return f"git_root {git_root.as_posix()} does not exist"
959 if not git_root.is_dir():
960 return f"git_root {git_root.as_posix()} is not a directory"
961 except OSError:
962 return f"git_root {git_root.as_posix()} cannot be read"
963 if not src_dir.is_relative_to(git_root):
964 return (
965 f"git_root {git_root.as_posix()} does not contain src_dir "
966 f"{src_dir.as_posix()}"
967 )
968 return None
969
970
971def git_root_warning(project: str, problem: str) -> str:
972 """The one warning text for a project's ignored ``git_root`` (``codelinks.git_root``)."""
973 return (
974 f"project {project!r}: {problem}; it is ignored, and the repository root is "
975 "detected from src_dir instead"
976 )
977
978
979def remote_url_pattern_warnings(config: CodeLinksConfig) -> list[str]:
980 """Why a project's remote URL pattern will not render as one link, if it will not.
981
982 The ``remote-url`` value is the pattern filled in, and Sphinx-Needs splits every
983 string-linked value on ``,`` and ``;`` before turning the parts into links. A
984 pattern containing either -- gitweb's ``?p=repo.git;a=blob;f={path}`` is the
985 classic -- therefore renders as several links, none of them right.
986 """
987 if not config.set_remote_url or not isinstance(config.projects, dict):
988 return []
989 warnings = []
990 for name, project_config in config.projects.items():
991 if not isinstance(project_config, dict):
992 continue
993 pattern = project_config.get("remote_url_pattern")
994 if isinstance(pattern, str) and ("," in pattern or ";" in pattern):
995 warnings.append(
996 f"Project {name!r}: remote_url_pattern {pattern!r} contains ',' or "
997 "';'. Sphinx-Needs splits string-linked values on ',' and ';', so this "
998 "pattern's links will not render as one link."
999 )
1000 return warnings
1001
1002
1003def _url_fields(config: CodeLinksConfig) -> set[str]:
1004 """The names of the URL fields the extension registers (when switched on)."""
1005 names: set[str] = set()
1006 if config.set_local_url:
1007 names.add(config.local_url_field)
1008 if config.set_remote_url:
1009 names.add(config.remote_url_field)
1010 return names
1011
1012
1013def need_id_refs_field(
1014 config: CodeLinksConfig, project_config: CodeLinksProjectConfigType
1015) -> str | None:
1016 """The field a project's ``@need-ids:`` references are attached to, or ``None``.
1017
1018 ubCode's gate: URLs are on (local or remote), references are extracted, the project
1019 has at least one marker, and its ``ref_url_field`` is not ``""``. A field that is not
1020 a string, or that is one of the URL fields, is a configuration error reported by
1021 :func:`check_project_configuration`, and attaches nothing.
1022 """
1023 if not (config.set_remote_url or config.set_local_url):
1024 return None
1025 field_name = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
1026 if not isinstance(field_name, str) or not field_name:
1027 return None
1028 if field_name in _url_fields(config):
1029 return None
1030 analyse_config = project_config.get("analyse_config")
1031 if analyse_config is None or not analyse_config.get_need_id_refs:
1032 return None
1033 if not analyse_config.need_id_refs_config.markers:
1034 return None
1035 return field_name
1036
1037
1038def need_id_refs_fields(config: CodeLinksConfig) -> dict[str, str]:
1039 """Each project whose references are attached, mapped to its field."""
1040 projects = config.projects
1041 if not isinstance(projects, dict):
1042 return {}
1043 fields_by_project: dict[str, str] = {}
1044 for name, project_config in projects.items():
1045 if not isinstance(project_config, dict):
1046 continue
1047 field_name = need_id_refs_field(config, project_config)
1048 if field_name is not None:
1049 fields_by_project[name] = field_name
1050 return fields_by_project
1051
1052
1053def check_configuration(config: CodeLinksConfig) -> list[str]:
1054 errors = []
1055 errors.extend(check_schema(config))
1056 errors.extend(check_project_configuration(config))
1057 return errors
1058
1059
1060def convert_src_discovery_config(
1061 config_dict: SourceDiscoverSectionConfigType | None,
1062) -> SourceDiscoverConfig:
1063 if config_dict:
1064 src_discover_dict = {
1065 key: (Path(value) if key == "src_dir" and isinstance(value, str) else value)
1066 for key, value in config_dict.items()
1067 }
1068 src_discover_config = SourceDiscoverConfig(**src_discover_dict) # ty: ignore[invalid-argument-type]
1069 else:
1070 src_discover_config = SourceDiscoverConfig()
1071
1072 return src_discover_config
1073
1074
1075def _validate_preprocessor_dict(preproc: dict[str, object]) -> None:
1076 """Validate the schema-less ``[preprocessor]`` TOML section.
1077
1078 The section has no ``TypedDict``, so a mistyped scalar would otherwise be
1079 coerced into garbage instead of reported: e.g. ``defines = "X"`` (a bare
1080 string) becomes ``list("X") == ["X"]`` — or worse, ``defines = "cpp17"``
1081 becomes ``["c", "p", "p", "1", "7"]`` → five bogus ``-D`` flags. Fail loud.
1082
1083 :param preproc: the raw ``[preprocessor]`` mapping from TOML.
1084 :raises TypeError: if a key has the wrong type.
1085 """
1086 for key in ("defines", "includes"):
1087 value = preproc.get(key)
1088 if value is not None and (
1089 not isinstance(value, list) or not all(isinstance(x, str) for x in value)
1090 ):
1091 raise TypeError(
1092 f"[preprocessor] {key} must be a list of strings, "
1093 f"got {type(value).__name__}: {value!r}"
1094 )
1095 for key in ("compile_commands", "std"):
1096 value = preproc.get(key)
1097 if value is not None and not isinstance(value, str):
1098 raise TypeError(
1099 f"[preprocessor] {key} must be a string, "
1100 f"got {type(value).__name__}: {value!r}"
1101 )
1102
1103
1104REMOVED_ANALYSE_KEYS: dict[str, str] = {
1105 "get_rst": "get_multiline_needs",
1106 "marked_rst": "[analyse.multiline_needs]",
1107}
1108"""``[analyse]`` keys of the removed marked-rst blocks -> what replaces each."""
1109
1110
1111def check_removed_analyse_keys(config_dict: AnalyseSectionConfigType) -> None:
1112 """Refuse the keys of the removed ``@rst`` blocks with one message naming the cure.
1113
1114 Any other unknown key keeps failing as before, in the dataclass constructor.
1115
1116 :raises TypeError: through the channel every other configuration error of the
1117 section takes (``typer.BadParameter`` in the CLI, the ``config-inited`` error of
1118 a Sphinx build).
1119 """
1120 found = [key for key in REMOVED_ANALYSE_KEYS if key in config_dict]
1121 if not found:
1122 return
1123 removed = " and ".join(
1124 f"'{key}'" if key == "get_rst" else f"'[analyse.{key}]'" for key in found
1125 )
1126 replacements = " and ".join(REMOVED_ANALYSE_KEYS[key] for key in found)
1127 raise TypeError(
1128 f"analyse: {removed} {'is' if len(found) == 1 else 'are'} no longer supported: "
1129 "the marked-rst blocks were replaced by multi-line needs (the @need and "
1130 f"@endneed markers); use {replacements} instead"
1131 )
1132
1133
1134def convert_analyse_config(
1135 config_dict: AnalyseSectionConfigType | None,
1136 src_discover: SourceDiscover | None = None,
1137) -> SourceAnalyseConfig:
1138 analyse_config_dict: SourceAnalyseConfigType = {}
1139 if config_dict:
1140 check_removed_analyse_keys(config_dict)
1141 for k, v in config_dict.items():
1142 if k not in {
1143 "oneline_comment_style",
1144 "need_id_refs",
1145 "multiline_needs",
1146 "preprocessor",
1147 }:
1148 # Convert string paths to Path objects
1149 if k in {"src_dir", "git_root"} and isinstance(v, str):
1150 analyse_config_dict[k] = Path(v)
1151 else:
1152 # dynamical assignment
1153 analyse_config_dict[k] = v # ty: ignore[invalid-key]
1154
1155 # Get oneline_comment_style configuration
1156 oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get(
1157 "oneline_comment_style"
1158 )
1159 oneline_comment_style: OneLineCommentStyle = (
1160 convert_oneline_comment_style_config(oneline_comment_style_dict)
1161 )
1162
1163 # Get need_id_refs configuration
1164 need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get(
1165 "need_id_refs"
1166 )
1167 need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict)
1168
1169 # Get multiline_needs configuration
1170 multiline_needs_config_dict: MultilineNeedsConfigType | None = config_dict.get(
1171 "multiline_needs"
1172 )
1173 multiline_needs_config = convert_multiline_needs_config(
1174 multiline_needs_config_dict
1175 )
1176
1177 analyse_config_dict["need_id_refs_config"] = need_id_refs_config
1178 analyse_config_dict["multiline_needs_config"] = multiline_needs_config
1179 analyse_config_dict["oneline_comment_style"] = oneline_comment_style
1180
1181 preprocessor_dict = config_dict.get("preprocessor")
1182 if preprocessor_dict is not None:
1183 # The preprocessor section has no TypedDict; its values are dynamic
1184 # TOML (typed ``object``), so validate the shapes up front -- a mistyped
1185 # scalar would otherwise coerce into garbage flags.
1186 _validate_preprocessor_dict(preprocessor_dict)
1187 analyse_config_dict["preprocessor"] = PreprocessorConfig(
1188 compile_commands=(
1189 Path(str(preprocessor_dict["compile_commands"]))
1190 if preprocessor_dict.get("compile_commands")
1191 else None
1192 ),
1193 defines=list(preprocessor_dict.get("defines", [])), # ty: ignore[invalid-argument-type]
1194 includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # ty: ignore[not-iterable]
1195 std=str(preprocessor_dict.get("std", DEFAULT_CPP_STD)),
1196 )
1197
1198 if src_discover:
1199 analyse_config_dict["src_files"] = src_discover.source_paths
1200 analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir
1201 try:
1202 analyse_config_dict["comment_type"] = CommentType(
1203 src_discover.src_discover_config.comment_type
1204 )
1205 except ValueError:
1206 # If invalid comment_type, keep the string value
1207 # Validation will catch this error later
1208 comment_type_str: str = src_discover.src_discover_config.comment_type
1209 analyse_config_dict["comment_type"] = comment_type_str # ty: ignore[invalid-assignment]
1210
1211 return SourceAnalyseConfig(**analyse_config_dict)
1212
1213
1214def convert_oneline_comment_style_config(
1215 config_dict: OneLineCommentStyleType | None,
1216) -> OneLineCommentStyle:
1217 if config_dict is None:
1218 oneline_comment_style = OneLineCommentStyle()
1219 else:
1220 try:
1221 oneline_comment_style = OneLineCommentStyle(**config_dict)
1222 except TypeError as e:
1223 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1224 return oneline_comment_style
1225
1226
1227def convert_need_id_refs_config(
1228 config_dict: NeedIdRefsConfigType | None,
1229) -> NeedIdRefsConfig:
1230 if not config_dict:
1231 need_id_refs_config = NeedIdRefsConfig()
1232 else:
1233 try:
1234 need_id_refs_config = NeedIdRefsConfig(**config_dict)
1235 except TypeError as e:
1236 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1237 return need_id_refs_config
1238
1239
1240def convert_multiline_needs_config(
1241 config_dict: MultilineNeedsConfigType | None,
1242) -> MultilineNeedsConfig:
1243 if not config_dict:
1244 return MultilineNeedsConfig()
1245 try:
1246 return MultilineNeedsConfig(**config_dict)
1247 except TypeError as e:
1248 raise TypeError(f"Invalid multiline_needs configuration: {e}") from e
1249
1250
1251def generate_project_configs(
1252 project_configs: dict[str, CodeLinksProjectConfigType],
1253) -> None:
1254 """Generate configs of source discover and analyse from their classes dynamically."""
1255 for project_config in project_configs.values():
1256 # overwrite the config into different types on purpose
1257 # covert dicts to their own classes
1258 src_discover_section: SourceDiscoverSectionConfigType | None = cast(
1259 SourceDiscoverSectionConfigType,
1260 project_config.get("source_discover"),
1261 )
1262 source_discover_config = convert_src_discovery_config(src_discover_section)
1263 project_config["source_discover_config"] = source_discover_config
1264
1265 analyse_section_config: AnalyseSectionConfigType | None = cast(
1266 AnalyseSectionConfigType, project_config.get("analyse")
1267 )
1268 analyse_config = convert_analyse_config(analyse_section_config)
1269 analyse_config.get_oneline_needs = True # force to get oneline_need
1270 # Copy comment_type from source_discover_config to analyse_config
1271 try:
1272 analyse_config.comment_type = CommentType(
1273 source_discover_config.comment_type
1274 )
1275 except ValueError:
1276 # If invalid comment_type, keep the string value
1277 # Validation will catch this error later
1278 analyse_config.comment_type = source_discover_config.comment_type # ty: ignore[invalid-assignment]
1279 project_config["analyse_config"] = analyse_config
Back to top