1import json
2import os
3from collections.abc import Generator
4from dataclasses import dataclass
5from pathlib import Path
6from typing import Any, cast
7
8from tree_sitter import Node as TreeSitterNode
9from tree_sitter import Parser, Query
10
11from sphinx_codelinks.analyse import multiline_parser, utils
12from sphinx_codelinks.analyse.models import (
13 MarkedContentType,
14 MultilineNeed,
15 NeedIdRefs,
16 OneLineNeed,
17 SourceComment,
18 SourceFile,
19 SourceMap,
20 WarningSubTypeEnum,
21)
22from sphinx_codelinks.analyse.oneline_parser import (
23 OnelineParserInvalidWarning,
24 docstring_tag,
25 jsdoc_tag,
26 oneline_parser,
27)
28from sphinx_codelinks.analyse.references import _relative_posix
29from sphinx_codelinks.config import (
30 UNIX_NEWLINE,
31 CommentCategory,
32 OneLineCommentStyle,
33 SourceAnalyseConfig,
34)
35from sphinx_codelinks.logger import get_logger
36from sphinx_codelinks.source_discover.config import CommentType
37
38logger = get_logger(__name__)
39
40# The last key of the marked content's sort. Two entries can start at one position
41# only with overlapping markers (a need-id marker that ends with the one-line start
42# sequence), which the configuration check does not refuse. Such a tie is listed
43# references, then one-line needs, then multi-line needs: an explicit rank, because
44# the type names sort the other way round. Two references from one comment tie on
45# every key and keep the marker order of the configuration (the sort is stable).
46_KIND_RANK: dict[MarkedContentType, int] = {
47 MarkedContentType.need_id_refs: 0,
48 MarkedContentType.need: 1,
49 MarkedContentType.multiline_need: 2,
50}
51
52
53def _char_column(src: bytes, byte_offset: int) -> int:
54 """The column, in characters, of ``byte_offset`` in the UTF-8 ``src``."""
55 line_start = src.rfind(b"\n", 0, byte_offset) + 1
56 return len(src[line_start:byte_offset].decode("utf-8", errors="replace"))
57
58
59def _count(n: int, noun: str) -> str:
60 """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
61 return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
62
63
64def _row_count(src_comment: SourceComment) -> int:
65 """The rows a comment covers: a Rust ``///`` node's text ends with its newline, which
66 starts no row of its own."""
67 return (src_comment.node.text or b"").rstrip(b"\n").count(b"\n") + 1
68
69
70#: How a tag warning names the tag it found.
71_TAG_NAMES = {
72 WarningSubTypeEnum.docstring_tag: "docstring tag",
73 WarningSubTypeEnum.jsdoc_tag: "JSDoc tag",
74}
75
76
77def _is_jsdoc(node: TreeSitterNode) -> bool:
78 """A JSDoc comment: a block comment opened by exactly ``/**``, as JSDoc reads one
79 (``/***`` and the empty ``/**/`` are not)."""
80 text = node.text or b""
81 return (
82 text.startswith(b"/**")
83 and not text.startswith(b"/***")
84 and not text.startswith(b"/**/")
85 )
86
87
88def _docstring_contents(
89 node: TreeSitterNode, column: int
90) -> list[tuple[str, int, int]]:
91 """The content of each string of a docstring statement, between its quotes.
92
93 Each with its 0-based row and its column, in characters: the quotes' column plus
94 the prefix letters' and the delimiter's length. ``column`` is the statement's.
95 """
96 raw = node.text or b""
97 contents = []
98 for string in node.named_children:
99 if string.type != "string":
100 continue
101 kinds = {child.type: child for child in string.children}
102 start, end = kinds.get("string_start"), kinds.get("string_end")
103 if start is None or end is None:
104 continue
105 begin = start.end_byte - node.start_byte
106 before = raw[:begin]
107 row_start = before.rfind(b"\n") + 1
108 content_column = len(before[row_start:].decode("utf-8", errors="replace"))
109 if row_start == 0:
110 content_column += column
111 contents.append(
112 (
113 raw[begin : end.start_byte - node.start_byte].decode(
114 "utf-8", errors="replace"
115 ),
116 start.end_point.row,
117 content_column,
118 )
119 )
120 return contents
121
122
123def _scanned_texts(
124 src_comment: SourceComment, claimed_rows: set[int] | None
125) -> list[tuple[str, int, int]]:
126 """What the one-line and reference extractors scan of a comment.
127
128 Each text comes with the 0-based row and the column, in characters, at which it
129 starts. A docstring is scanned as its content, without the quotes; a block
130 comment without its closing ``*/``, and a legacy ``<!-- … -->`` comment without
131 its ``-->``, which are not marker text. The rows a multi-line need claims are
132 blanked.
133 """
134 node = src_comment.node
135 if getattr(node, "type", None) == CommentCategory.docstring:
136 texts = _docstring_contents(node, src_comment.column)
137 else:
138 text = node.text.decode("utf-8") if node.text else ""
139 if text.startswith("/*") and text.endswith("*/"):
140 text = text[:-2]
141 elif text.startswith("<!--") and text.endswith("-->"):
142 text = text[:-3]
143 texts = [(text, node.start_point.row, src_comment.column)]
144 if claimed_rows:
145 # a block's lines are its own: no one-line need and no reference in them
146 texts = [
147 (multiline_parser.blank_rows(text, row, claimed_rows), row, column)
148 for text, row, column in texts
149 ]
150 return texts
151
152
153@dataclass
154class AnalyseWarning:
155 file_path: str
156 lineno: int
157 msg: str
158 type: str
159 sub_type: str
160
161
162class SourceAnalyse:
163 def __init__(
164 self,
165 analyse_config: SourceAnalyseConfig,
166 *,
167 name: str = "",
168 ) -> None:
169 self.name = name
170 self.analyse_config = analyse_config
171 self.src_files: list[SourceFile] = []
172 self.src_comments: list[SourceComment] = []
173 self.need_id_refs: list[NeedIdRefs] = []
174 self.oneline_needs: list[OneLineNeed] = []
175 self.multiline_needs: list[MultilineNeed] = []
176 self.all_marked_content: list[NeedIdRefs | OneLineNeed | MultilineNeed] = []
177 # Use explicitly configured git_root if provided, otherwise auto-detect
178 if self.analyse_config.git_root is not None:
179 self.git_root: Path | None = self.analyse_config.git_root.resolve()
180 else:
181 self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
182 self.git_remote_url: str | None = (
183 utils.get_remote_url(self.git_root) if self.git_root else None
184 )
185 self.git_commit_rev: str | None = (
186 utils.get_current_rev(self.git_root) if self.git_root else None
187 )
188 self.project_path: Path = self.git_root or self.analyse_config.src_dir
189 self.warnings: list[AnalyseWarning] = []
190 """The analysis' warnings, one-line and multi-line alike, in extraction order."""
191 # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
192 # database is read once per run instead of once per source file.
193 self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
194
195 @property
196 def oneline_warnings(self) -> list[AnalyseWarning]:
197 """The old name of :attr:`warnings`, kept for one release; read-only."""
198 return self.warnings
199
200 def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:
201 """Load source files and extract their content."""
202 for src_path in self.analyse_config.src_files:
203 if not utils.is_text_file(src_path):
204 continue
205 with src_path.open("r", encoding="utf-8", newline="") as f:
206 # Normalize all line endings to Unix LF
207 text = f.read()
208 text = text.replace("\r\n", "\n").replace("\r", "\n")
209 yield src_path, text.encode("utf-8")
210
211 def create_src_objects(self) -> None:
212 comment_type = self.analyse_config.comment_type
213 # One (parser, query) pair per distinct grammar actually needed, built
214 # lazily so a parser is never rebuilt per file. Every comment type
215 # except TypeScript uses a single grammar for the whole run;
216 # TypeScript alone varies its grammar per file (utils.ts_grammar_key)
217 # because a legacy TypeScript-only cast parses as JSX under the wrong
218 # grammar — see the CommentType.ts branch of utils.init_tree_sitter.
219 parser_cache: dict[str, tuple[Parser, Query]] = {}
220
221 for src_path, src_string in self.get_src_strings():
222 # `comment_type` is normally a CommentType member, but a few call
223 # sites carry it as a plain (possibly invalid) str instead — see
224 # SourceAnalyseConfig.comment_type — so key on `str(comment_type)`
225 # rather than `.value`, which only the enum has.
226 cache_key = (
227 utils.ts_grammar_key(src_path)
228 if comment_type == CommentType.ts
229 else str(comment_type)
230 )
231 if cache_key not in parser_cache:
232 parser_cache[cache_key] = utils.init_tree_sitter(comment_type, src_path)
233 parser, query = parser_cache[cache_key]
234
235 comments: list[TreeSitterNode] | None = utils.extract_comments(
236 src_string, parser, query
237 )
238 if not comments:
239 continue
240 src_comments: list[SourceComment] = [
241 SourceComment(node, _char_column(src_string, node.start_byte))
242 for node in comments
243 ]
244
245 src_file = SourceFile(src_path.absolute())
246 src_file.add_comments(src_comments)
247 if self.analyse_config.get_multiline_needs:
248 src_file.lines = src_string.decode("utf-8").split(UNIX_NEWLINE)
249 self.src_files.append(src_file)
250 self.src_comments.extend(src_comments)
251
252 def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
253 """Return the parsed compile_commands.json for ``db_path``, memoized.
254
255 The database is read and parsed once per run instead of once per source
256 file (O(files x entries) -> O(entries)). A present-but-malformed or
257 unreadable database is warned once and cached as ``None`` so callers fall
258 back to the configured defines without re-reading or re-warning per file.
259 """
260 from sphinx_codelinks.analyse.preproc import compile_db
261
262 if db_path in self._flags_map_cache:
263 return self._flags_map_cache[db_path]
264 try:
265 flags = compile_db.load_flags_map(db_path)
266 except (OSError, ValueError, TypeError) as exc:
267 logger.warning(
268 f"codelinks: failed to read {db_path} ({exc}); "
269 f"falling back to the configured defines"
270 )
271 self._flags_map_cache[db_path] = None
272 return None
273 self._flags_map_cache[db_path] = flags
274 return flags
275
276 def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
277 from sphinx_codelinks.analyse.preproc import compile_db
278
279 # `run()` calls this (via create_src_objects_libclang) only when
280 # `preprocessor is not None`, but keep the guard: it narrows the type for
281 # the checker and is a cheap defense (an `assert` would trip bandit S101).
282 preproc = self.analyse_config.preprocessor
283 if preproc is None:
284 return []
285 db_path = preproc.compile_commands
286 if db_path is None:
287 db_path = compile_db.find_compile_db(src_path, self.project_path)
288 if db_path is not None and db_path.is_file():
289 flags = self._flags_map_for(db_path)
290 if flags is None:
291 # Present-but-malformed/unreadable DB (warned once in the helper):
292 # fall back to the configured defines so extraction still runs.
293 return compile_db.defines_to_args(
294 preproc.defines, preproc.includes, preproc.std
295 )
296 args = flags.get(src_path.absolute().resolve())
297 if args is not None:
298 return args
299 # Absent from the DB. compile_commands.json lists only compiled
300 # translation units, never headers — so a header here is parsed
301 # standalone with the global defines (one run = one variant). A
302 # compiled source absent from the build is skipped (spec §3.3).
303 if compile_db.is_translation_unit_source(src_path):
304 return None
305 return compile_db.defines_to_args(
306 preproc.defines, preproc.includes, preproc.std
307 )
308 # No readable DB. `find_compile_db` only ever returns a real file, so a
309 # non-None `db_path` that reaches here is an explicit `compile_commands`
310 # path that is not a readable file (typo/missing): warn and fall back
311 # rather than skip silently. A genuinely absent DB (db_path is None) just
312 # falls back to the manual defines applied globally.
313 if db_path is not None:
314 logger.warning(
315 f"codelinks: compile_commands path {db_path} is not a readable "
316 f"file; falling back to the configured defines"
317 )
318 return compile_db.defines_to_args(
319 preproc.defines, preproc.includes, preproc.std
320 )
321
[docs]322 # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
323 def create_src_objects_libclang(self) -> None:
324 from sphinx_codelinks.analyse.preproc import (
325 libclang_parser,
326 loader,
327 )
328
329 # Resolve the exception via the loader (never a direct ``import
330 # clang.cindex``) so a missing ``libclang`` extra still surfaces the
331 # loader's friendly install hint rather than a bare ImportError.
332 translation_unit_load_error = (
333 loader.load_clang_cindex().TranslationUnitLoadError
334 )
335
336 for src_path in self.analyse_config.src_files:
337 if not utils.is_text_file(src_path):
338 continue
339 args = self._resolve_preproc_args(src_path)
340 if args is None:
341 logger.debug(
342 f"codelinks: skipping {src_path} — not found in compile_commands.json"
343 )
344 continue
345 try:
346 comments = libclang_parser.extract_active_comments(src_path, args)
347 except translation_unit_load_error:
348 # Last-resort guard. Standalone parses pin ``-x`` to match the
349 # ``-std`` (see defines_to_args), so the common case — a ``.h``/
350 # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
351 # markers extract. This only fires if libclang still cannot load
352 # the file as a translation unit at all; skip it rather than
353 # aborting the whole run. Surfaced as a ``warning`` so the silent
354 # data-loss (a file's markers dropped) is visible — accepting that
355 # this fails ``sphinx-build -W``.
356 logger.warning(
357 f"codelinks: skipping {src_path} — libclang could not load it "
358 f"as a translation unit"
359 )
360 continue
361 if not comments:
362 continue
363 # ``c`` is a LibclangComment duck-typing the tree-sitter Node
364 # interface SourceComment reads (``.text`` / ``.start_point.row``);
365 # the Node-only path (find_associated_scope) is guarded by
366 # ``is_libclang`` so it never runs on these. The cast is that duck typing,
367 # which is why ty, seeing two unrelated classes, has to be told.
368 src_comments = [
369 SourceComment(
370 cast("TreeSitterNode", c), # ty: ignore[disjoint-cast]
371 c.column,
372 )
373 for c in comments
374 ]
375 src_file = SourceFile(src_path.absolute())
376 src_file.add_comments(src_comments)
377 if self.analyse_config.get_multiline_needs:
378 # multi-line needs read what precedes a comment on its row from
379 # the row itself
380 text = src_path.read_text(encoding="utf-8", errors="replace")
381 text = text.replace("\r\n", "\n").replace("\r", "\n")
382 src_file.lines = text.split(UNIX_NEWLINE)
383 self.src_files.append(src_file)
384 self.src_comments.extend(src_comments)
385
386 def extract_marker(
387 self,
388 text: str,
389 ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
390 row_offset = 0
391 for line in text.split(UNIX_NEWLINE):
392 for marker in self.analyse_config.need_id_refs_config.markers:
393 marker_idx = line.find(marker)
394 if marker_idx == -1:
395 continue
396 after_marker = line[marker_idx + len(marker) :]
397 markered_text = after_marker.strip()
398 if markered_text.endswith("*/"):
399 # The end of a one-line block comment, not an id.
400 markered_text = markered_text[:-2].rstrip()
401 need_ids = markered_text.replace(",", " ").split()
402 if not need_ids:
403 continue
404 start_column = (
405 marker_idx
406 + len(marker)
407 + len(after_marker)
408 - len(after_marker.lstrip())
409 )
410 end_column = start_column + len(markered_text)
411 yield marker, need_ids, row_offset, start_column, end_column
412 row_offset += 1
413
[docs]414 # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
415 def extract_anchors(
416 self,
417 text: str,
418 filepath: Path,
419 tagged_scope: TreeSitterNode | None,
420 src_comment: SourceComment,
421 first_row: int | None = None,
422 first_column: int | None = None,
423 ) -> list[NeedIdRefs]:
424 """Extract need-ids-refs from a comment.
425
426 ``text`` starts at ``first_row`` and ``first_column`` (by default the
427 comment's own row and column).
428 """
429 if first_row is None:
430 first_row = src_comment.node.start_point.row
431 if first_column is None:
432 first_column = src_comment.column
433 anchors: list[NeedIdRefs] = []
434 for (
435 marker,
436 need_ids,
437 row_offset,
438 start_column,
439 end_column,
440 ) in self.extract_marker(text):
441 lineno = first_row + row_offset + 1
442 line_column = first_column if row_offset == 0 else 0
443 remote_url = self.git_remote_url
444 if self.git_remote_url and self.git_commit_rev:
445 remote_url = utils.form_https_url(
446 self.git_remote_url,
447 self.git_commit_rev,
448 self.project_path,
449 filepath,
450 lineno,
451 )
452 source_map: SourceMap = {
453 "start": {
454 "row": lineno - 1,
455 "column": line_column + start_column,
456 },
457 "end": {
458 "row": lineno - 1,
459 "column": line_column + end_column,
460 },
461 }
462 anchors.append(
463 NeedIdRefs(
464 filepath,
465 remote_url,
466 source_map,
467 src_comment,
468 tagged_scope,
469 need_ids,
470 marker,
471 )
472 )
473 return anchors
474
475 def _is_need_id_refs_line(self, line: str) -> bool:
476 """Whether a comment line is an ``@need-ids:`` reference rather than a need.
477
478 A line whose text, after comment decoration and whitespace, starts with a
479 configured need-id-refs marker is a reference, and never a one-line need --
480 ubCode's precedence. Without it the default one-line start sequence ``@`` also
481 matched ``// @need-ids: A, B`` and made a need titled ``need-ids: A`` with the id
482 ``B``. A line with anything alphanumeric before the marker is not a reference
483 line here (``// see @need-ids: X`` -- the one-line parser ignores it anyway, and
484 the references on it are still extracted), and a line holding a one-line need
485 before the marker (``[[…]] @need-ids: X``) keeps its need: only a line that
486 STARTS with the marker is withheld from the one-line parser.
487 """
488 for marker in self.analyse_config.need_id_refs_config.markers:
489 marker_idx = line.find(marker)
490 if marker_idx != -1 and not any(
491 char.isalnum() for char in line[:marker_idx]
492 ):
493 return True
494 return False
495
496 def extract_oneline_need(
497 self,
498 text: str,
499 src_comment: SourceComment,
500 oneline_comment_style: OneLineCommentStyle,
501 first_row: int | None = None,
502 ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
503 if first_row is None:
504 first_row = src_comment.node.start_point.row
505 # every line counts as terminated, the last one too
506 lines = [f"{line}{UNIX_NEWLINE}" for line in text.split(UNIX_NEWLINE)]
507 row_offset = 0
508
509 # Only a Python docstring can hold docstring tags; a ``#`` comment never does.
510 in_docstring = (
511 getattr(src_comment.node, "type", None) == CommentCategory.docstring
512 )
513 # Only a TypeScript or JavaScript JSDoc comment can hold JSDoc tags.
514 in_jsdoc = self.analyse_config.comment_type == CommentType.ts and _is_jsdoc(
515 src_comment.node
516 )
517 for line in lines:
518 if self._is_need_id_refs_line(line):
519 row_offset += 1
520 continue
521 start_sequence = oneline_comment_style.start_sequence
522 tag, kind, holders = None, WarningSubTypeEnum.docstring_tag, "docstrings"
523 if in_docstring:
524 tag = docstring_tag(line, start_sequence)
525 elif in_jsdoc and start_sequence == "@":
526 # ``@`` is JSDoc's own sigil: under any other start sequence a JSDoc
527 # line is read as any other line is
528 tag = jsdoc_tag(line, start_sequence)
529 kind, holders = WarningSubTypeEnum.jsdoc_tag, "JSDoc comments"
530 if tag is not None and not oneline_parser(line, oneline_comment_style):
531 # a tag line the one-line parser would ignore stays silent
532 row_offset += 1
533 continue
534 if tag is not None:
535 if src_comment.source_file:
536 self.warnings.append(
537 AnalyseWarning(
538 str(src_comment.source_file.filepath),
539 first_row + row_offset + 1,
540 f"'{start_sequence}{tag}' is a {_TAG_NAMES[kind]}, not a "
541 "one-line need; use a start sequence that "
542 f"{holders} do not contain",
543 MarkedContentType.need,
544 kind.value,
545 )
546 )
547 row_offset += 1
548 continue
549 resolved = oneline_parser(line, oneline_comment_style)
550 if not resolved:
551 row_offset += 1
552 continue
553 if isinstance(resolved, OnelineParserInvalidWarning):
554 if not src_comment.source_file:
555 row_offset += 1
556 continue
557 lineno = first_row + row_offset + 1
558 warning = AnalyseWarning(
559 str(src_comment.source_file.filepath),
560 lineno,
561 resolved.msg,
562 MarkedContentType.need,
563 resolved.sub_type.value,
564 )
565 self.warnings.append(warning)
566 row_offset += 1
567 continue
568 yield resolved, row_offset
569 row_offset += 1
570
[docs]571 # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
572 def extract_oneline_needs(
573 self,
574 text: str,
575 filepath: Path,
576 tagged_scope: TreeSitterNode | None,
577 src_comment: SourceComment,
578 oneline_comment_style: OneLineCommentStyle,
579 first_row: int | None = None,
580 first_column: int | None = None,
581 ) -> list[OneLineNeed]:
582 """Extract the one-line needs of a comment.
583
584 ``text`` starts at ``first_row`` and ``first_column`` (by default the
585 comment's own row and column).
586 """
587 if first_row is None:
588 first_row = src_comment.node.start_point.row
589 if first_column is None:
590 first_column = src_comment.column
591 row_offset = 0
592 oneline_needs = []
593 for resolved, row_offset in self.extract_oneline_need(
594 text, src_comment, oneline_comment_style, first_row
595 ):
596 lineno = first_row + row_offset + 1
597 line_column = first_column if row_offset == 0 else 0
598 start_column = line_column + cast("int", resolved["start_column"])
599 end_column = line_column + cast("int", resolved["end_column"])
600 remote_url = self.git_remote_url
601 if self.git_remote_url and self.git_commit_rev:
602 remote_url = utils.form_https_url(
603 self.git_remote_url,
604 self.git_commit_rev,
605 self.project_path,
606 filepath,
607 lineno,
608 )
609 source_map: SourceMap = {
610 "start": {
611 "row": lineno - 1,
612 "column": start_column,
613 },
614 "end": {
615 "row": lineno - 1,
616 "column": end_column,
617 },
618 }
619 del resolved["start_column"]
620 del resolved["end_column"]
621 oneline_needs.append(
622 OneLineNeed(
623 filepath,
624 remote_url,
625 source_map,
626 src_comment,
627 tagged_scope,
628 resolved, # int arguments were deleted # ty: ignore[invalid-argument-type]
629 )
630 )
631 return oneline_needs
632
[docs]633 # @Extract multi-line needs from comment runs, IMPL_MLN_1, impl, [FE_MULTILINE_NEEDS]
634 def extract_multiline_needs(self) -> dict[int, set[int]]:
635 """Extract the multi-line needs of every file, and the rows their blocks claim.
636
637 Runs before the other extractors: the rows a block claims (and those of a
638 refused header, consumed to its close) are hidden from them.
639
640 :return: For each comment holding claimed rows (keyed by ``id()`` of its
641 :class:`SourceComment`), those 0-based rows.
642 """
643 claimed: dict[int, set[int]] = {}
644 config = self.analyse_config.multiline_needs_config
645 for src_file in self.src_files:
646 for run in multiline_parser.form_runs(
647 src_file.src_comments, src_file.lines
648 ):
649 result = multiline_parser.parse_run(
650 run.lines,
651 config,
652 leaderless_block=run.leaderless_block,
653 mixed_leaders=run.mixed_leaders,
654 )
655 for issue in result.issues:
656 self.warnings.append(
657 AnalyseWarning(
658 str(src_file.filepath),
659 issue.row + 1,
660 issue.msg,
661 MarkedContentType.multiline_need,
662 issue.kind.value,
663 )
664 )
665 for block in result.blocks:
666 self.multiline_needs.append(
667 self._multiline_need(block, run, src_file.filepath)
668 )
669 if not result.claimed_rows:
670 continue
671 for src_comment in run.comments:
672 first = src_comment.node.start_point.row
673 rows = result.claimed_rows.intersection(
674 range(first, first + _row_count(src_comment))
675 )
676 if rows:
677 claimed.setdefault(id(src_comment), set()).update(rows)
678 return claimed
679
680 def _multiline_need(
681 self,
682 block: multiline_parser.ParsedBlock,
683 run: multiline_parser.CommentRun,
684 filepath: Path,
685 ) -> MultilineNeed:
686 """The record of one parsed block (see :class:`MultilineNeed`)."""
687 src_comment = next(
688 (
689 comment
690 for comment in run.comments
691 if comment.node.start_point.row
692 <= block.open_row
693 < comment.node.start_point.row + _row_count(comment)
694 ),
695 run.comments[0],
696 )
697 tagged_scope: TreeSitterNode | None = None
698 if not getattr(src_comment.node, "is_libclang", False):
699 tagged_scope = utils.find_associated_scope(
700 src_comment.node, self.analyse_config.comment_type
701 )
702 lineno = block.open_row + 1
703 remote_url = self.git_remote_url
704 if self.git_remote_url and self.git_commit_rev:
705 remote_url = utils.form_https_url(
706 self.git_remote_url,
707 self.git_commit_rev,
708 self.project_path,
709 filepath,
710 lineno,
711 )
712 source_map: SourceMap = {
713 "start": {"row": block.open_row, "column": block.open_col},
714 "end": {"row": block.close_row, "column": block.end_col},
715 }
716 content_start = (
717 {"line": block.content_start[0] + 1, "col": block.content_start[1]}
718 if block.content_start is not None
719 else None
720 )
721 scope = (
722 {
723 "kind": tagged_scope.type,
724 "start": tagged_scope.start_point.row + 1,
725 "end": tagged_scope.end_point.row + 1,
726 }
727 if tagged_scope is not None
728 else None
729 )
730 source: dict[str, Any] = {
731 "project": self.name,
732 "path": _relative_posix(filepath, self.project_path),
733 "root": "git" if self.git_root is not None else "src_dir",
734 "commit": self.git_commit_rev,
735 "start": {"line": lineno, "col": block.open_col},
736 "end": {"line": block.close_row + 1, "col": block.end_col},
737 "content_start": content_start,
738 "option_lines": dict(block.option_rows),
739 "scope": scope,
740 }
741 return MultilineNeed(
742 filepath,
743 remote_url,
744 source_map,
745 src_comment,
746 tagged_scope,
747 block.need,
748 block.markup,
749 source,
750 )
751
752 def extract_marked_content(self) -> None:
753 claimed: dict[int, set[int]] = {}
754 if self.analyse_config.get_multiline_needs:
755 claimed = self.extract_multiline_needs()
756 for src_comment in self.src_comments:
757 if not src_comment.node.text:
758 continue
759 filepath = (
760 src_comment.source_file.filepath if src_comment.source_file else None
761 )
762 if not filepath:
763 continue
764 if getattr(src_comment.node, "is_libclang", False):
765 tagged_scope: TreeSitterNode | None = None
766 else:
767 tagged_scope = utils.find_associated_scope(
768 src_comment.node, self.analyse_config.comment_type
769 )
770 for text, first_row, first_column in _scanned_texts(
771 src_comment, claimed.get(id(src_comment))
772 ):
773 if self.analyse_config.get_need_id_refs:
774 self.need_id_refs.extend(
775 self.extract_anchors(
776 text,
777 filepath,
778 tagged_scope,
779 src_comment,
780 first_row,
781 first_column,
782 )
783 )
784 if self.analyse_config.get_oneline_needs:
785 self.oneline_needs.extend(
786 self.extract_oneline_needs(
787 text,
788 filepath,
789 tagged_scope,
790 src_comment,
791 self.analyse_config.oneline_comment_style,
792 first_row,
793 first_column,
794 )
795 )
796
797 def merge_marked_content(self) -> None:
798 """List the marked content in source order: by file, row and column (#2150).
799
800 Tree-sitter hands a file's comments over in an order that differs between
801 runs, so the column is what keeps two entries of one row in a stable order.
802 ``oneline_needs`` is sorted too, by row, then file, then column: ``src-trace``
803 creates and renders the needs in its order, so of two markers with one id in
804 one file the leftmost is the one created, and on one row the needs of two files
805 keep the file order they had before the column was a key -- for a caller that
806 passes the files in discovery order, as every production caller does.
807 """
808 # The file component is the key discovery sorts the files by
809 # (``SourceDiscover``), not the ``Path``: a ``Path`` compares by parts, so
810 # ``a/b.cpp`` would sort before ``a-c.cpp``, which discovery lists first.
811 self.oneline_needs.sort(
812 key=lambda x: (
813 x.source_map["start"]["row"],
814 os.path.normcase(os.path.normpath(x.filepath)),
815 x.source_map["start"]["column"],
816 )
817 )
818 self.all_marked_content.extend(self.need_id_refs)
819 self.all_marked_content.extend(self.oneline_needs)
820 self.all_marked_content.extend(self.multiline_needs)
821 self.all_marked_content.sort(
822 key=lambda x: (
823 x.filepath,
824 x.source_map["start"]["row"],
825 x.source_map["start"]["column"],
826 _KIND_RANK[x.type],
827 )
828 )
829
830 def dump_marked_content(self, outdir: Path) -> None:
831 output_path = outdir / "marked_content.json"
832 output_path.parent.mkdir(parents=True, exist_ok=True)
833 to_dump = [
834 marked_content.to_dict() for marked_content in self.all_marked_content
835 ]
836 with output_path.open("w") as f:
837 json.dump(to_dump, f)
838
839 def run(self, *, log_summary: bool = True) -> None:
840 """Parse the source files and extract their markers.
841
842 :param log_summary: Whether to print the per-project summary line; a caller
843 that prints its own (the configuration pass of the Sphinx extension) passes
844 ``False``.
845 """
846 if (
847 self.analyse_config.preprocessor is not None
848 and self.analyse_config.comment_type == CommentType.cpp
849 ):
850 self.create_src_objects_libclang()
851 else:
852 self.create_src_objects()
853 self.extract_marked_content()
854 self.merge_marked_content()
855 if log_summary:
856 self.log_summary()
857
858 def log_summary(self, extra: str = "") -> None:
859 """Emit a per-project marker (default-visible) plus a -v breakdown.
860
861 :param extra: Appended to the default-visible line (the ``src-trace``
862 directive's count of the one-line needs it did not create).
863 """
864 label = f"codelinks [{self.name}]" if self.name else "codelinks"
865 logger.info(
866 f"{label}: {_count(len(self.src_files), 'file')}, "
867 f"{_count(len(self.all_marked_content), 'marker')}{extra}"
868 )
869 logger.debug(
870 f"{label}: {_count(len(self.src_comments), 'comment')}, "
871 f"{_count(len(self.oneline_needs), 'oneline need')}, "
872 f"{_count(len(self.need_id_refs), 'id-ref')}, "
873 f"{_count(len(self.multiline_needs), 'multi-line need')}"
874 )
Back to top