Merge branch 'mauro' into docs-mw

Mauro's work to include documentation from our Python modules.  His cover
letter follows:

This is an extended version of:
    https://lore.kernel.org/linux-doc/cover.1768488832.git.mchehab+huawei@kernel.org/

It basically adds everything we currently have inside libs/tool/python
to "tools" book inside documentation.

This version should be independent of the other series yet to be merged,
(including the jobserver one).

The vast amount of changes here are docstring cleanups and additions.
They mainly consists on:

- ensuring that every phrase will end with a period, making it uniform
  along all files;
- cleaning ups to better uniform docstrings;
- variable descriptions now use "#:" markup, as it allows autodoc to
  add them inside the documentation;
- added some missing docstrings;
- some new blank lines at comments to make ReST syntax parser happy;
- add a couple of sphinx markups (mainly, code blocks).

Most of those are minor changes, affecting only comments.

It also has one patch per libarary type, adding them to docs.

For kernel-doc, I did the cleanups first, as there is one code block
inside tools/lib/python/kdoc/latex_fonts.py that would cause a Sphinx
crash without such markups.

The series actually starts with 3 fixes:

- avoid "*" markups on indexes with deep> 3 to override text
- a variable rename to stop abusing doctree name
- don't rely on cwd to get Documentation/ location

patch 4 adds support to document scripts either at:
    - tools/
    - scripts/

patch 5 contains a CSS to better display autodoc html output.

For those who want to play with documentation, documenting a python
file is very simple. All it takes is to use:

    .. automodule:: lib.python.<dir+name>

Usually, we add a couple of control members to it to adjust
the desired documentation scope (add/remove members, showing class
inheritance, showing members that currently don't have
docstrings, etc). That's why we're using:

    .. automodule:: lib.python.kdoc.enrich_formatter
       :members:
       :show-inheritance:
       :undoc-members:

(and similar) inside tools/kdoc*.rst.

autodoc allows filtering in/out members, file docstrings, etc.

It also allows documenting just some members or functions with
directives like:

    ..autofunction:
    ..automember:

Sphinx also has a helper script to generate .rst files with
documentation:

    $ sphinx-apidoc -o foobar tools/lib/python/

which can be helpful to discover what should be documented,
although changes are needed to use what it produces.

Documentation/conf.py

@@ -13,10 +13,15 @@ from  textwrap import dedent
 
 import sphinx
 
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-sys.path.insert(0, os.path.abspath("sphinx"))
+# Location of Documentation/ directory
+kern_doc_dir = os.path.dirname(os.path.abspath(__file__))
+
+# Add location of Sphinx extensions
+sys.path.insert(0, os.path.join(kern_doc_dir, "sphinx"))
+
+# Allow sphinx.ext.autodoc to document files at tools and scripts
+sys.path.append(os.path.join(kern_doc_dir, "..", "tools"))
+sys.path.append(os.path.join(kern_doc_dir, "..", "scripts"))
 
 # Minimal supported version
 needs_sphinx = "3.4.3"
@@ -32,9 +37,6 @@ else:
     # Include patterns that don't contain directory names, in glob format
     include_patterns = ["**.rst"]
 
-# Location of Documentation/ directory
-doctree = os.path.abspath(".")
-
 # Exclude of patterns that don't contain directory names, in glob format.
 exclude_patterns = []
 
@@ -73,7 +75,7 @@ def config_init(app, config):
     # setup include_patterns dynamically
     if has_include_patterns:
         for p in dyn_include_patterns:
-            full = os.path.join(doctree, p)
+            full = os.path.join(kern_doc_dir, p)
 
             rel_path = os.path.relpath(full, start=app.srcdir)
             if rel_path.startswith("../"):
@@ -83,7 +85,7 @@ def config_init(app, config):
 
     # setup exclude_patterns dynamically
     for p in dyn_exclude_patterns:
-        full = os.path.join(doctree, p)
+        full = os.path.join(kern_doc_dir, p)
 
         rel_path = os.path.relpath(full, start=app.srcdir)
         if rel_path.startswith("../"):
@@ -95,7 +97,7 @@ def config_init(app, config):
     # of the app.srcdir. Add them here
 
     # Handle the case where SPHINXDIRS is used
-    if not os.path.samefile(doctree, app.srcdir):
+    if not os.path.samefile(kern_doc_dir, app.srcdir):
         # Add a tag to mark that the build is actually a subproject
         tags.add("subproject")
 
@@ -154,6 +156,7 @@ extensions = [
     "maintainers_include",
     "parser_yaml",
     "rstFlatTable",
+    "sphinx.ext.autodoc",
     "sphinx.ext.autosectionlabel",
     "sphinx.ext.ifconfig",
     "translations",

Documentation/sphinx-static/custom.css

@@ -30,6 +30,9 @@ img.logo {
     margin-bottom: 20px;
 }
 
+/* The default is to use -1em, wich makes it override text */
+li { text-indent: 0em; }
+
 /*
  * Parameters for the display of function prototypes and such included
  * from C source files.
@@ -40,6 +43,15 @@ dl.function dt { margin-left: 10em; text-indent: -10em; }
 dt.sig-object { font-size: larger; }
 div.kernelindent { margin-left: 2em; margin-right: 4em; }
 
+/*
+ * Parameters for the display of function prototypes and such included
+ * from Python source files.
+ */
+dl.py { margin-top: 2em; background-color: #ecf0f3; }
+dl.py.class { margin-left: 2em; text-indent: -2em; padding-left: 2em; }
+dl.py.method, dl.py.attribute { margin-left: 2em; text-indent: -2em; }
+dl.py li, pre { text-indent: 0em; padding-left: 0 !important; }
+
 /*
  * Tweaks for our local TOC
  */

Documentation/tools/feat.rst

@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+====================================
+Documentation features parser module
+====================================
+
+.. automodule:: lib.python.feat.parse_features
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/index.rst

@@ -12,6 +12,7 @@ more additions are needed here:
 
    rtla/index
    rv/index
+   python
 
 .. only::  subproject and html
 

Documentation/tools/jobserver.rst

@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+Job server module
+=================
+
+.. automodule:: lib.python.jobserver
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kabi.rst

@@ -0,0 +1,13 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+Kernel ABI documentation tool modules
+=====================================
+
+.. toctree::
+   :maxdepth: 2
+
+   kabi_parser
+   kabi_regex
+   kabi_symbols
+   kabi_helpers

Documentation/tools/kabi_helpers.rst

@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+Ancillary classes
+=================
+
+.. automodule:: lib.python.abi.helpers
+   :members:
+   :member-order: bysource
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kabi_parser.rst

@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=====================================
+Kernel ABI documentation parser class
+=====================================
+
+.. automodule:: lib.python.abi.abi_parser
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kabi_regex.rst

@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================
+ABI regex search symbol class
+=============================
+
+.. automodule:: lib.python.abi.abi_regex
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kabi_symbols.rst

@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=========================================
+System ABI documentation validation class
+=========================================
+
+.. automodule:: lib.python.abi.system_symbols
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kdoc.rst

@@ -0,0 +1,12 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==================
+Kernel-doc modules
+==================
+
+.. toctree::
+   :maxdepth: 2
+
+   kdoc_parser
+   kdoc_output
+   kdoc_ancillary

Documentation/tools/kdoc_ancillary.rst

@@ -0,0 +1,46 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=================
+Ancillary classes
+=================
+
+Argparse formatter class
+========================
+
+.. automodule:: lib.python.kdoc.enrich_formatter
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Regular expression class handler
+================================
+
+.. automodule:: lib.python.kdoc.kdoc_re
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+
+Chinese, Japanese and Korean variable fonts handler
+===================================================
+
+.. automodule:: lib.python.kdoc.latex_fonts
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Kernel C file include logic
+===========================
+
+.. automodule:: lib.python.kdoc.parse_data_structs
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Python version ancillary methods
+================================
+
+.. automodule:: lib.python.kdoc.python_version
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/kdoc_output.rst

@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=======================
+Kernel-doc output stage
+=======================
+
+Output handler for man pages and ReST
+=====================================
+
+.. automodule:: lib.python.kdoc.kdoc_output
+   :members:
+   :show-inheritance:
+   :undoc-members:
+

Documentation/tools/kdoc_parser.rst

@@ -0,0 +1,29 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=======================
+Kernel-doc parser stage
+=======================
+
+File handler classes
+====================
+
+.. automodule:: lib.python.kdoc.kdoc_files
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Parsed item data class
+======================
+
+.. automodule:: lib.python.kdoc.kdoc_item
+   :members:
+   :show-inheritance:
+   :undoc-members:
+
+Parser classes and methods
+==========================
+
+.. automodule:: lib.python.kdoc.kdoc_parser
+   :members:
+   :show-inheritance:
+   :undoc-members:

Documentation/tools/python.rst

@@ -0,0 +1,13 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+================
+Python libraries
+================
+
+.. toctree::
+   :maxdepth: 4
+
+   jobserver
+   feat
+   kdoc
+   kabi

tools/lib/python/abi/abi_parser.py

@@ -21,14 +21,17 @@ from abi.helpers import AbiDebug, ABI_DIR
 
 
 class AbiParser:
-    """Main class to parse ABI files"""
+    """Main class to parse ABI files."""
 
+    #: Valid tags at Documentation/ABI.
     TAGS = r"(what|where|date|kernelversion|contact|description|users)"
+
+    #: ABI elements that will auto-generate cross-references.
     XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)"
 
     def __init__(self, directory, logger=None,
                  enable_lineno=False, show_warnings=True, debug=0):
-        """Stores arguments for the class and initialize class vars"""
+        """Stores arguments for the class and initialize class vars."""
 
         self.directory = directory
         self.enable_lineno = enable_lineno
@@ -65,7 +68,7 @@ class AbiParser:
         self.re_xref_node = re.compile(self.XREF)
 
     def warn(self, fdata, msg, extra=None):
-        """Displays a parse error if warning is enabled"""
+        """Displays a parse error if warning is enabled."""
 
         if not self.show_warnings:
             return
@@ -77,7 +80,7 @@ class AbiParser:
         self.log.warning(msg)
 
     def add_symbol(self, what, fname, ln=None, xref=None):
-        """Create a reference table describing where each 'what' is located"""
+        """Create a reference table describing where each 'what' is located."""
 
         if what not in self.what_symbols:
             self.what_symbols[what] = {"file": {}}
@@ -92,7 +95,7 @@ class AbiParser:
             self.what_symbols[what]["xref"] = xref
 
     def _parse_line(self, fdata, line):
-        """Parse a single line of an ABI file"""
+        """Parse a single line of an ABI file."""
 
         new_what = False
         new_tag = False
@@ -264,7 +267,7 @@ class AbiParser:
             self.warn(fdata, "Unexpected content", line)
 
     def parse_readme(self, nametag, fname):
-        """Parse ABI README file"""
+        """Parse ABI README file."""
 
         nametag["what"] = ["Introduction"]
         nametag["path"] = "README"
@@ -282,7 +285,7 @@ class AbiParser:
                 nametag["description"] += line
 
     def parse_file(self, fname, path, basename):
-        """Parse a single file"""
+        """Parse a single file."""
 
         ref = f"abi_file_{path}_{basename}"
         ref = self.re_unprintable.sub("_", ref).strip("_")
@@ -348,7 +351,7 @@ class AbiParser:
                     self.add_symbol(what=w, fname=fname, xref=fdata.key)
 
     def _parse_abi(self, root=None):
-        """Internal function to parse documentation ABI recursively"""
+        """Internal function to parse documentation ABI recursively."""
 
         if not root:
             root = self.directory
@@ -377,7 +380,7 @@ class AbiParser:
                 self.parse_file(name, path, basename)
 
     def parse_abi(self, root=None):
-        """Parse documentation ABI"""
+        """Parse documentation ABI."""
 
         self._parse_abi(root)
 
@@ -385,7 +388,7 @@ class AbiParser:
             self.log.debug(pformat(self.data))
 
     def desc_txt(self, desc):
-        """Print description as found inside ABI files"""
+        """Print description as found inside ABI files."""
 
         desc = desc.strip(" \t\n")
 
@@ -393,7 +396,7 @@ class AbiParser:
 
     def xref(self, fname):
         """
-        Converts a Documentation/ABI + basename into a ReST cross-reference
+        Converts a Documentation/ABI + basename into a ReST cross-reference.
         """
 
         xref = self.file_refs.get(fname)
@@ -403,7 +406,7 @@ class AbiParser:
             return xref
 
     def desc_rst(self, desc):
-        """Enrich ReST output by creating cross-references"""
+        """Enrich ReST output by creating cross-references."""
 
         # Remove title markups from the description
         # Having titles inside ABI files will only work if extra
@@ -459,7 +462,7 @@ class AbiParser:
 
     def doc(self, output_in_txt=False, show_symbols=True, show_file=True,
             filter_path=None):
-        """Print ABI at stdout"""
+        """Print ABI at stdout."""
 
         part = None
         for key, v in sorted(self.data.items(),
@@ -549,7 +552,7 @@ class AbiParser:
             yield (msg, file_ref[0][0], ln)
 
     def check_issues(self):
-        """Warn about duplicated ABI entries"""
+        """Warn about duplicated ABI entries."""
 
         for what, v in self.what_symbols.items():
             files = v.get("file")
@@ -575,7 +578,7 @@ class AbiParser:
             self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f))
 
     def search_symbols(self, expr):
-        """ Searches for ABI symbols """
+        """ Searches for ABI symbols."""
 
         regex = re.compile(expr, re.I)
 

tools/lib/python/abi/abi_regex.py

@@ -16,10 +16,22 @@ from abi.abi_parser import AbiParser
 from abi.helpers import AbiDebug
 
 class AbiRegex(AbiParser):
-    """Extends AbiParser to search ABI nodes with regular expressions"""
+    """
+    Extends AbiParser to search ABI nodes with regular expressions.
 
-    # Escape only ASCII visible characters
+    There some optimizations here to allow a quick symbol search:
+    instead of trying to place all symbols altogether an doing linear
+    search which is very time consuming, create a tree with one depth,
+    grouping similar symbols altogether.
+
+    Yet, sometimes a full search will be needed, so we have a special branch
+    on such group tree where other symbols are placed.
+    """
+
+    #: Escape only ASCII visible characters.
     escape_symbols = r"([\x21-\x29\x2b-\x2d\x3a-\x40\x5c\x60\x7b-\x7e])"
+
+    #: Special group for other nodes.
     leave_others = "others"
 
     # Tuples with regular expressions to be compiled and replacement data
@@ -88,13 +100,15 @@ class AbiRegex(AbiParser):
         # Recover plus characters
         (re.compile(r"\xf7"), "+"),
     ]
+
+    #: Regex to check if the symbol name has a number on it.
     re_has_num = re.compile(r"\\d")
 
-    # Symbol name after escape_chars that are considered a devnode basename
+    #: Symbol name after escape_chars that are considered a devnode basename.
     re_symbol_name =  re.compile(r"(\w|\\[\.\-\:])+$")
 
-    # List of popular group names to be skipped to minimize regex group size
-    # Use AbiDebug.SUBGROUP_SIZE to detect those
+    #: List of popular group names to be skipped to minimize regex group size
+    #: Use AbiDebug.SUBGROUP_SIZE to detect those.
     skip_names = set(["devices", "hwmon"])
 
     def regex_append(self, what, new):
@@ -148,7 +162,7 @@ class AbiRegex(AbiParser):
     def get_regexes(self, what):
         """
         Given an ABI devnode, return a list of all regular expressions that
-        may match it, based on the sub-groups created by regex_append()
+        may match it, based on the sub-groups created by regex_append().
         """
 
         re_list = []

tools/lib/python/abi/helpers.py

@@ -13,26 +13,28 @@ ABI_DIR = "Documentation/ABI/"
 class AbiDebug:
     """Debug levels"""
 
-    WHAT_PARSING = 1
-    WHAT_OPEN = 2
-    DUMP_ABI_STRUCTS = 4
-    UNDEFINED = 8
-    REGEX = 16
-    SUBGROUP_MAP = 32
-    SUBGROUP_DICT = 64
-    SUBGROUP_SIZE = 128
-    GRAPH = 256
-
+    WHAT_PARSING = 1        #: Enable debug parsing logic.
+    WHAT_OPEN = 2           #: Enable debug messages on file open.
+    DUMP_ABI_STRUCTS = 4    #: Enable debug for ABI parse data.
+    UNDEFINED = 8           #: Enable extra undefined symbol data.
+    REGEX = 16              #: Enable debug for what to regex conversion.
+    SUBGROUP_MAP = 32       #: Enable debug for symbol regex subgroups
+    SUBGROUP_DICT = 64      #: Enable debug for sysfs graph tree variable.
+    SUBGROUP_SIZE = 128     #: Enable debug of search groups.
+    GRAPH = 256             #: Display ref tree graph for undefined symbols.
 
+#: Helper messages for each debug variable
 DEBUG_HELP = """
-1  - enable debug parsing logic
-2  - enable debug messages on file open
-4  - enable debug for ABI parse data
-8  - enable extra debug information to identify troubles
-     with ABI symbols found at the local machine that
-     weren't found on ABI documentation (used only for
-     undefined subcommand)
-16 - enable debug for what to regex conversion
-32 - enable debug for symbol regex subgroups
-64 - enable debug for sysfs graph tree variable
+1   - enable debug parsing logic
+2   - enable debug messages on file open
+4   - enable debug for ABI parse data
+8   - enable extra debug information to identify troubles
+      with ABI symbols found at the local machine that
+      weren't found on ABI documentation (used only for
+      undefined subcommand)
+16  - enable debug for what to regex conversion
+32  - enable debug for symbol regex subgroups
+64  - enable debug for sysfs graph tree variable
+128 - enable debug of search groups
+256 - enable displaying refrence tree graphs for undefined symbols.
 """

tools/lib/python/abi/system_symbols.py

@@ -18,11 +18,11 @@ from random import shuffle
 from abi.helpers import AbiDebug
 
 class SystemSymbols:
-    """Stores arguments for the class and initialize class vars"""
+    """Stores arguments for the class and initialize class vars."""
 
     def graph_add_file(self, path, link=None):
         """
-        add a file path to the sysfs graph stored at self.root
+        add a file path to the sysfs graph stored at self.root.
         """
 
         if path in self.files:
@@ -43,7 +43,7 @@ class SystemSymbols:
         self.files.add(path)
 
     def print_graph(self, root_prefix="", root=None, level=0):
-        """Prints a reference tree graph using UTF-8 characters"""
+        """Prints a reference tree graph using UTF-8 characters."""
 
         if not root:
             root = self.root
@@ -173,7 +173,7 @@ class SystemSymbols:
         self._walk(sysfs)
 
     def check_file(self, refs, found):
-        """Check missing ABI symbols for a given sysfs file"""
+        """Check missing ABI symbols for a given sysfs file."""
 
         res_list = []
 
@@ -214,7 +214,7 @@ class SystemSymbols:
         return res_list
 
     def _ref_interactor(self, root):
-        """Recursive function to interact over the sysfs tree"""
+        """Recursive function to interact over the sysfs tree."""
 
         for k, v in root.items():
             if isinstance(v, dict):
@@ -232,7 +232,7 @@ class SystemSymbols:
 
 
     def get_fileref(self, all_refs, chunk_size):
-        """Interactor to group refs into chunks"""
+        """Interactor to group refs into chunks."""
 
         n = 0
         refs = []
@@ -250,7 +250,7 @@ class SystemSymbols:
 
     def check_undefined_symbols(self, max_workers=None, chunk_size=50,
                                 found=None, dry_run=None):
-        """Seach ABI for sysfs symbols missing documentation"""
+        """Seach ABI for sysfs symbols missing documentation."""
 
         self.abi.parse_abi()
 

tools/lib/python/feat/parse_features.py

@@ -21,14 +21,25 @@ class ParseFeature:
     from it.
     """
 
+    #: feature header string.
     h_name = "Feature"
+
+    #: Kernel config header string.
     h_kconfig = "Kconfig"
+
+    #: description header string.
     h_description = "Description"
+
+    #: subsystem header string.
     h_subsys = "Subsystem"
+
+    #: status header string.
     h_status = "Status"
+
+    #: architecture header string.
     h_arch = "Architecture"
 
-    # Sort order for status. Others will be mapped at the end.
+    #: Sort order for status. Others will be mapped at the end.
     status_map = {
         "ok":   0,
         "TODO": 1,
@@ -40,7 +51,7 @@ class ParseFeature:
 
     def __init__(self, prefix, debug=0, enable_fname=False):
         """
-        Sets internal variables
+        Sets internal variables.
         """
 
         self.prefix = prefix
@@ -63,11 +74,13 @@ class ParseFeature:
         self.msg = ""
 
     def emit(self, msg="", end="\n"):
+        """Helper function to append a new message for feature output."""
+
         self.msg += msg + end
 
     def parse_error(self, fname, ln, msg, data=None):
         """
-        Displays an error message, printing file name and line
+        Displays an error message, printing file name and line.
         """
 
         if ln:
@@ -82,7 +95,7 @@ class ParseFeature:
             print("", file=sys.stderr)
 
     def parse_feat_file(self, fname):
-        """Parses a single arch-support.txt feature file"""
+        """Parses a single arch-support.txt feature file."""
 
         if os.path.isdir(fname):
             return
@@ -204,7 +217,7 @@ class ParseFeature:
         self.max_size_arch_with_header = self.max_size_arch + len(self.h_arch)
 
     def parse(self):
-        """Parses all arch-support.txt feature files inside self.prefix"""
+        """Parses all arch-support.txt feature files inside self.prefix."""
 
         path = os.path.expanduser(self.prefix)
 
@@ -281,7 +294,7 @@ class ParseFeature:
 
     def output_feature(self, feat):
         """
-        Output a feature on all architectures
+        Output a feature on all architectures.
         """
 
         title = f"Feature {feat}"
@@ -331,7 +344,7 @@ class ParseFeature:
 
     def matrix_lines(self, desc_size, max_size_status, header):
         """
-        Helper function to split element tables at the output matrix
+        Helper function to split element tables at the output matrix.
         """
 
         if header: