generate_docs.py

import sys
import warnings
from pydoc import ModuleScanner


def scan_modules():
    """taken from the source code of help('modules')

    https://github.com/python/cpython/blob/63298930fb531ba2bb4f23bc3b915dbf1e17e9e1/Lib/pydoc.py#L2178"""
    modules = {}

    def callback(path, modname, desc, modules=modules):
        if modname and modname[-9:] == ".__init__":
            modname = modname[:-9] + " (package)"
        if modname.find(".") < 0:
            modules[modname] = 1

    def onerror(modname):
        callback(None, modname, None)

    with warnings.catch_warnings():
        # ignore warnings from importing deprecated modules
        warnings.simplefilter("ignore")
        ModuleScanner().run(callback, onerror=onerror)
    return list(modules.keys())


def import_module(module_name):
    import io
    from contextlib import redirect_stdout

    # Importing modules causes ('Constant String', 2, None, 4) and
    # "Hello world!" to be printed to stdout.
    f = io.StringIO()
    with warnings.catch_warnings(), redirect_stdout(f):
        # ignore warnings caused by importing deprecated modules
        warnings.filterwarnings("ignore", category=DeprecationWarning)
        try:
            module = __import__(module_name)
        except Exception as e:
            return e
    return module


def is_child(module, item):
    import inspect

    item_mod = inspect.getmodule(item)
    return item_mod is module


def traverse(module, names, item):
    import inspect

    has_doc = inspect.ismodule(item) or inspect.isclass(item) or inspect.isbuiltin(item)
    if has_doc and isinstance(item.__doc__, str):
        yield names, item.__doc__
    attr_names = dir(item)
    for name in attr_names:
        if name in [
            "__class__",
            "__dict__",
            "__doc__",
            "__objclass__",
            "__name__",
            "__qualname__",
            "__annotations__",
        ]:
            continue
        try:
            attr = getattr(item, name)
        except AttributeError:
            assert name == "__abstractmethods__", name
            continue

        if module is item and not is_child(module, attr):
            continue

        is_type_or_module = (type(attr) is type) or (type(attr) is type(__builtins__))
        new_names = names.copy()
        new_names.append(name)

        if item == attr:
            pass
        elif not inspect.ismodule(item) and inspect.ismodule(attr):
            pass
        elif is_type_or_module:
            yield from traverse(module, new_names, attr)
        elif (
            callable(attr)
            or not issubclass(type(attr), type)
            or type(attr).__name__ in ("getset_descriptor", "member_descriptor")
        ):
            if inspect.isbuiltin(attr):
                yield new_names, attr.__doc__
        else:
            assert False, (module, new_names, attr, type(attr).__name__)


def traverse_all():
    for module_name in scan_modules():
        if module_name in ("this", "antigravity"):
            continue
        module = import_module(module_name)
        if hasattr(module, "__cached__"):  # python module
            continue
        yield from traverse(module, [module_name], module)


def docs():
    return ((".".join(names), doc) for names, doc in traverse_all())


if __name__ == "__main__":
    import sys
    import json

    try:
        out_path = sys.argv[1]
    except IndexError:
        out_path = "-"

    def dump(docs):
        yield "[\n"
        for name, doc in docs:
            if doc is None:
                yield f"    ({json.dumps(name)}, None),\n"
            else:
                yield f"    ({json.dumps(name)}, Some({json.dumps(doc)})),\n"
        yield "]\n"

    out_file = open(out_path, "w") if out_path != "-" else sys.stdout
    out_file.writelines(dump(docs()))