1import configparser
2from collections.abc import ByteString, Callable
3from pathlib import Path
4from urllib.request import pathname2url
5
6from giturlparse import parse
7from tree_sitter import Language, Parser, Point, Query, QueryCursor
8from tree_sitter import Node as TreeSitterNode
9
10from sphinx_codelinks.config import CommentCategory
11from sphinx_codelinks.logger import get_logger
12from sphinx_codelinks.source_discover.config import CommentType
13
14# Language-specific node types for scope detection.
15#
16# YAML and JSONC are intentionally absent. They are data formats, not code, so a
17# comment associates with the surrounding data structure (key/value pair, list
18# item, or scalar) rather than with an enclosing or following declaration. That
19# needs a different algorithm (inline same-row association first, scalar targets,
20# grammar-specific traversal), implemented in find_yaml_associated_structure and
21# find_jsonc_associated_structure and dispatched from find_associated_scope.
22# Those bespoke finders never read this table (only find_next_scope and
23# find_enclosing_scope do), so an entry here would be dead.
24SCOPE_NODE_TYPES = {
[docs] 25 # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
26 CommentType.python: {"function_definition", "class_definition"},
[docs] 27 # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
28 CommentType.cpp: {"function_definition", "class_specifier", "struct_specifier"},
29 CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
[docs] 30 # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST]
31 CommentType.rust: {
32 "function_item",
33 "struct_item",
34 "enum_item",
35 "impl_item",
36 "trait_item",
37 "mod_item",
38 },
[docs] 39 # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
40 CommentType.go: {
41 "function_declaration",
42 "method_declaration",
43 "type_declaration",
44 "type_spec",
45 },
[docs] 46 # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
47 CommentType.bash: {"function_definition"},
48}
49
50logger = get_logger(__name__)
51
52GIT_HOST_URL_TEMPLATE = {
53 "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
54 "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
55}
56
57PYTHON_QUERY = """
58 ; Match comments
59 (comment) @comment
60
61 ; Match docstrings inside modules, functions, or classes
62 (module (expression_statement (string)) @comment)
63 (function_definition (block (expression_statement (string)) @comment))
64 (class_definition (block (expression_statement (string)) @comment))
65 """
66CPP_QUERY = """(comment) @comment"""
67C_SHARP_QUERY = """(comment) @comment"""
68YAML_QUERY = """(comment) @comment"""
69RUST_QUERY = """
70 (line_comment) @comment
71 (block_comment) @comment
72"""
[docs] 73# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
74GO_QUERY = """
75 (comment) @comment
76"""
77JSONC_QUERY = """(comment) @comment"""
[docs] 78# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
79BASH_QUERY = """(comment) @comment"""
80
81# JSON value node types that can be associated with a comment.
82JSON_STRUCTURE_TYPES = {
83 "pair",
84 "object",
85 "array",
86 "string",
87 "number",
88 "true",
89 "false",
90 "null",
91}
92
93
94def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
95 """Return True if file is likely text, False if binary."""
96 try:
97 with filepath.open("rb") as f:
98 chunk = f.read(sample_size)
99 # Quick binary heuristic: null byte present
100 if b"\x00" in chunk:
101 return False
102 # Try UTF-8 decode on the sample
103 chunk.decode("utf-8")
104 return True
105 except UnicodeDecodeError:
106 return False
107
108
[docs]109# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
110def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
111 if comment_type == CommentType.cpp:
112 import tree_sitter_cpp
113
114 parsed_language = Language(tree_sitter_cpp.language())
115 query = Query(parsed_language, CPP_QUERY)
116 elif comment_type == CommentType.python:
117 import tree_sitter_python
118
119 parsed_language = Language(tree_sitter_python.language())
120 query = Query(parsed_language, PYTHON_QUERY)
121 elif comment_type == CommentType.cs:
122 import tree_sitter_c_sharp
123
124 parsed_language = Language(tree_sitter_c_sharp.language())
125 query = Query(parsed_language, C_SHARP_QUERY)
126 elif comment_type == CommentType.yaml:
127 import tree_sitter_yaml
128
129 parsed_language = Language(tree_sitter_yaml.language())
130 query = Query(parsed_language, YAML_QUERY)
131 elif comment_type == CommentType.rust:
132 import tree_sitter_rust
133
134 parsed_language = Language(tree_sitter_rust.language())
135 query = Query(parsed_language, RUST_QUERY)
136 elif comment_type == CommentType.go:
137 import tree_sitter_go
138
139 parsed_language = Language(tree_sitter_go.language())
140 query = Query(parsed_language, GO_QUERY)
141 elif comment_type == CommentType.jsonc:
142 import tree_sitter_json
143
144 parsed_language = Language(tree_sitter_json.language())
145 query = Query(parsed_language, JSONC_QUERY)
146 elif comment_type == CommentType.bash:
147 import tree_sitter_bash
148
149 parsed_language = Language(tree_sitter_bash.language())
150 query = Query(parsed_language, BASH_QUERY)
151 else:
152 raise ValueError(f"Unsupported comment style: {comment_type}")
153 parser = Parser(parsed_language)
154 return parser, query
155
156
157def wrap_read_callable_point(
158 src_string: ByteString,
159) -> Callable[[int, Point], ByteString]:
160 def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
161 return src_string[byte_offset : byte_offset + 1]
162
163 return read_callable_byte_offset
164
165
[docs]166# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
167def extract_comments(
168 src_string: ByteString, parser: Parser, query: Query
169) -> list[TreeSitterNode] | None:
170 """Get all comments from source files by tree-sitter."""
171 read_point_fn = wrap_read_callable_point(src_string)
172 tree = parser.parse(read_point_fn)
173 query_cursor = QueryCursor(query)
174 captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
175
176 return captures.get("comment")
177
178
179def find_enclosing_scope(
180 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
181) -> TreeSitterNode | None:
182 """Find the enclosing scope of a comment."""
183 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
184 current: TreeSitterNode = node
185 while current:
186 if current.type in scope_types:
187 return current
188 current: TreeSitterNode | None = current.parent # required for node traversal
189 return None
190
191
192def find_next_scope(
193 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
194) -> TreeSitterNode | None:
195 """Find the next scope of a comment."""
196 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
197 current: TreeSitterNode = node
198 while current:
199 if current.type in scope_types:
200 return current
201 current: TreeSitterNode | None = (
202 current.next_named_sibling
203 ) # required for node traversal
204 if current and current.type == "block":
205 for child in current.named_children:
206 if child.type in scope_types:
207 return child
208 return None
209
210
211def _find_yaml_structure_in_block_node(
212 block_node: TreeSitterNode,
213) -> TreeSitterNode | None:
214 """Find YAML structure elements within a block_node."""
215 for grandchild in block_node.named_children:
216 if grandchild.type == "block_mapping":
217 for ggchild in grandchild.named_children:
218 if ggchild.type == "block_mapping_pair":
219 return ggchild
220 elif grandchild.type == "block_sequence":
221 for ggchild in grandchild.named_children:
222 if ggchild.type == "block_sequence_item":
223 return ggchild
224 return None
225
226
227def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
228 """Find the next YAML structure element after the comment node."""
229 current = node.next_named_sibling
230 while current:
231 if current.type in {
232 "block_mapping_pair",
233 "block_sequence_item",
234 "flow_mapping",
235 "flow_sequence",
236 }:
237 return current
238 if current.type == "document":
239 for child in current.named_children:
240 if child.type == "block_node":
241 result = _find_yaml_structure_in_block_node(child)
242 if result:
243 return result
244 if current.type == "block_node":
245 result = _find_yaml_structure_in_block_node(current)
246 if result:
247 return result
248 current = current.next_named_sibling
249 return None
250
251
252def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
253 """Find a previous named sibling that is on the same row as the comment.
254
255 Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
256 """
257 comment_row = node.start_point.row
258 current = node.prev_named_sibling
259
260 while current:
261 # Check if this sibling ends on the same row as the comment starts
262 # This indicates it's an inline comment
263 if current.end_point.row == comment_row:
264 return current
265 # If we find a sibling that ends before the comment row, we can stop
266 # as we won't find any siblings on the same row going backwards
267 if current.end_point.row < comment_row:
268 break
269 current = current.prev_named_sibling
270
271 return None
272
273
274def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
275 """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
276 # First, check if this is an inline comment by looking for a previous sibling on the same row
277 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
278 if prev_sibling_same_row:
279 return prev_sibling_same_row
280
281 # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
282 structure = find_yaml_next_structure(node)
283 if structure:
284 return structure
285
286 # If no next sibling found, traverse up to find parent structure
287 parent = node.parent
288 while parent:
289 if parent.type in {"block_mapping_pair", "block_sequence_item"}:
290 return parent
291 parent = parent.parent
292
293 return None
294
295
[docs]296# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
297def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
298 """Find the JSON structure (key/value pair, value, list item) for a comment.
299
300 JSON is data rather than code, so association follows the same intent as YAML:
301 an inline comment belongs to the value on its row, a leading comment belongs to
302 the following structure, otherwise it belongs to the enclosing structure.
303 """
304 # Inline comment: a value/pair on the same row, before the comment
305 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
306 if prev_sibling_same_row:
307 return prev_sibling_same_row
308
309 # Leading comment: the next structure following the comment
310 current = node.next_named_sibling
311 while current:
312 if current.type in JSON_STRUCTURE_TYPES:
313 return current
314 current = current.next_named_sibling
315
316 # Otherwise: the enclosing structure
317 parent = node.parent
318 while parent:
319 if parent.type in {"pair", "object", "array"}:
320 return parent
321 parent = parent.parent
322
323 return None
324
325
326def find_associated_scope(
327 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
328) -> TreeSitterNode | None:
329 """Find the associated scope of a comment."""
330 if comment_type == CommentType.yaml:
331 # YAML uses different structure association logic
332 return find_yaml_associated_structure(node)
333
334 if comment_type == CommentType.jsonc:
335 # JSONC uses data-aware structure association logic
336 return find_jsonc_associated_structure(node)
337
338 if node.type == CommentCategory.docstring:
339 # Only for python's docstring
340 return find_enclosing_scope(node, comment_type)
341 # General comments regardless of comment types
342 associated_scope = find_next_scope(node, comment_type)
343 if not associated_scope:
344 associated_scope = find_enclosing_scope(node, comment_type)
345 return associated_scope
346
347
348def _git_dir(path: Path) -> Path | None:
349 """The git directory for a checkout rooted at ``path``, or None if it is not one.
350
351 In a plain checkout ``.git`` is a directory. In a LINKED WORKTREE it is a file
352 holding ``gitdir: <path>`` -- git's documented gitfile format -- and the real
353 directory is elsewhere, under the main repository's ``.git/worktrees/<name>``.
354 Treating the file as "not a repository" is what made every worktree of this
355 repository look unversioned (useblocks/sphinx-codelinks#106).
356 """
357 dot_git = path / ".git"
358 if dot_git.is_dir():
359 return dot_git
360 if dot_git.is_file():
361 content = dot_git.read_text(encoding="utf-8", errors="replace").strip()
362 if content.startswith("gitdir:"):
363 target = Path(content.split(":", 1)[1].strip())
364 if not target.is_absolute():
365 target = (path / target).resolve()
366 if target.is_dir():
367 return target
368 return None
369
370
371def _git_common_dir(git_dir: Path) -> Path:
372 """The repository-wide git directory that ``git_dir`` belongs to.
373
374 A linked worktree's git directory carries a ``commondir`` file naming the main
375 repository's, and that is where ``config`` and every shared ref live -- only
376 per-worktree state (``HEAD``, ``refs/bisect``, ``refs/worktree``) is local. A plain
377 checkout has no ``commondir`` and the two are the same directory.
378 """
379 commondir = git_dir / "commondir"
380 if not commondir.is_file():
381 return git_dir
382 target = Path(commondir.read_text(encoding="utf-8").strip())
383 if not target.is_absolute():
384 target = (git_dir / target).resolve()
385 return target
386
387
388def find_git_root(src_dir: Path) -> Path | None:
389 """The git root :func:`locate_git_root` finds, without its warning when none."""
390 current = src_dir.resolve()
391 parents = list(current.parents)
392 parents.append(current)
393 for parent in parents:
394 if _git_dir(parent) is not None:
395 return parent
396 return None
397
398
399def locate_git_root(src_dir: Path) -> Path | None:
400 """Traverse upwards to find git root."""
401 found = find_git_root(src_dir)
402 if found is not None:
403 return found
404 logger.warning(
405 f"git root is not found in the parent of {src_dir}",
406 subtype="git_root",
407 location=str(src_dir),
408 )
409 return None
410
411
412def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
413 """Get remote url from the git directory's config."""
414 git_dir = _git_dir(git_root)
415 # a worktree's remotes are the main repository's, so `config` is in the common dir
416 config_path = (
417 _git_common_dir(git_dir) if git_dir is not None else git_root / ".git"
418 ) / "config"
419 if not config_path.exists():
420 logger.warning(
421 f"{config_path} does not exist",
422 subtype="git_config",
423 location=str(config_path),
424 )
425 return None
426
427 config = configparser.ConfigParser(allow_no_value=True, strict=False)
428 config.read(config_path)
429 section = f'remote "{remote_name}"'
430 if section in config and "url" in config[section]:
431 url: str = config[section]["url"]
432 return url
433 logger.warning(
434 f"remote-url is not found in {config_path}",
435 subtype="git_remote",
436 location=str(config_path),
437 )
438 return None
439
440
441def _packed_ref(common_dir: Path, ref: str) -> str | None:
442 """The sha ``packed-refs`` records for ``ref``, or None.
443
444 A ref has no file of its own once ``git gc`` (or ``git pack-refs``) has run, which
445 happens unattended on any long-lived checkout -- so a loose-file lookup alone reports
446 "no rev" for a perfectly ordinary repository. The file is one ``<sha> <refname>`` per
447 line; ``^<sha>`` lines are the peeled targets of annotated tags and are skipped, as
448 is the leading ``#`` header.
449 """
450 packed = common_dir / "packed-refs"
451 if not packed.is_file():
452 return None
453 for line in packed.read_text(encoding="utf-8", errors="replace").splitlines():
454 if not line or line.startswith(("#", "^")):
455 continue
456 sha, _, name = line.partition(" ")
457 if name.strip() == ref:
458 return sha
459 return None
460
461
462def get_current_rev(git_root: Path) -> str | None:
463 """Get current commit rev from the git directory's HEAD."""
464 # HEAD is per-worktree, so it is read from the worktree's OWN git directory
465 git_dir = _git_dir(git_root) or git_root / ".git"
466 head_path = git_dir / "HEAD"
467 if not head_path.exists():
468 logger.warning(
469 f"{head_path} does not exist",
470 subtype="git_head",
471 location=str(head_path),
472 )
473 return None
474 head_content = head_path.read_text().strip()
475 if not head_content.startswith("ref: "):
476 # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
477 # directly, which is exactly the rev we want.
478 return head_content
479
480 ref = head_content.split(":", 1)[1].strip()
481 common_dir = _git_common_dir(git_dir)
482 ref_path = git_dir / ref
483 if not ref_path.exists():
484 # every branch is a SHARED ref, so in a worktree it lives in the common
485 # directory; only refs/bisect and refs/worktree are per-worktree
486 ref_path = common_dir / ref
487 if ref_path.exists():
488 return ref_path.read_text().strip()
489 packed = _packed_ref(common_dir, ref)
490 if packed is not None:
491 return packed
492 logger.warning(
493 f"{ref_path} does not exist",
494 subtype="git_ref",
495 location=str(ref_path),
496 )
497 return None
498
499
500def form_https_url(
501 git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
502) -> str | None:
503 parsed_url = parse(git_url)
504 template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
505 if not template:
506 logger.warning(
507 f"Unsupported Git host: {parsed_url.platform}",
508 subtype="git_host",
509 )
510 return git_url
511 https_url = template.format(
512 owner=parsed_url.owner,
513 repo=parsed_url.repo,
514 rev=rev,
515 path=pathname2url(str(filepath.absolute().relative_to(project_path))),
516 lineno=str(lineno),
517 )
518 return https_url
Back to top