CPython docs database for module items

Author: youknowone

Cargo.lock

@@ -717,8 +717,6 @@ def wrapper():
         with self.assertRaises(AttributeError):
             functools.update_wrapper(wrapper, f, assign, update)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     @support.requires_docstrings
     @unittest.skipIf(sys.flags.optimize >= 2,
                      "Docstrings are omitted with -O2 and above")

Lib/test/test_functools.py

@@ -18,6 +18,7 @@ proc-macro2 = "1.0"
 rustpython-compiler = { path = "../compiler/porcelain", version = "0.1.1" }
 rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
 maplit = "1.0"
-once_cell = "1.3.1"
+once_cell = "1.8.0"
 textwrap = "0.13.4"
 indexmap = "^1"
+serde_json = "1.0.68"

derive/Cargo.toml

@@ -0,0 +1,27 @@
+pub(crate) type Result = std::result::Result<Option<&'static str>, ()>;
+
+pub(crate) fn try_read(path: &str) -> Result {
+    static DATABASE: once_cell::sync::OnceCell<std::collections::HashMap<String, Option<String>>> =
+        once_cell::sync::OnceCell::new();
+    let db = DATABASE.get_or_init(|| {
+        let raw = include_str!("../docs.json");
+        serde_json::from_str(raw).expect("docs.json must be a valid json file")
+    });
+    let data = db.get(path).ok_or(())?;
+    Ok(data.as_ref().map(|s| s.as_str()))
+}
+
+pub(crate) fn try_module_item(module: &str, item: &str) -> Result {
+    try_read(&format!("{}.{}", module, item))
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_module_item() {
+        let doc = try_module_item("array", "_array_reconstructor").unwrap();
+        assert!(doc.is_some());
+    }
+}

derive/docs.json

@@ -13,6 +13,7 @@ mod error;
 mod util;
 
 mod compile_bytecode;
+mod doc;
 mod from_args;
 mod pyclass;
 mod pymodule;

derive/src/doc.rs

@@ -261,11 +261,19 @@ impl ModuleItem for FunctionItem {
         let sig_doc = text_signature(func.sig(), &py_name);
 
         let item = {
-            let doc = args.attrs.doc().map_or_else(TokenStream::new, |mut doc| {
-                doc = format!("{}\n--\n\n{}", sig_doc, doc);
-                quote!(.with_doc(#doc.to_owned(), &vm.ctx))
-            });
             let module = args.module_name();
+            let doc = args.attrs.doc().or_else(|| {
+                crate::doc::try_module_item(module, &py_name)
+                    .ok() // TODO: doc must exist at least one of code or CPython
+                    .flatten()
+                    .map(str::to_owned)
+            });
+            let doc = if let Some(doc) = doc {
+                format!("{}\n--\n\n{}", sig_doc, doc)
+            } else {
+                sig_doc
+            };
+            let doc = quote!(.with_doc(#doc.to_owned(), &vm.ctx));
             let new_func = quote_spanned!(ident.span()=>
                 vm.ctx.make_funcdef(#py_name, #ident)
                     #doc

derive/src/lib.rs

@@ -0,0 +1,101 @@
+import sys
+import warnings
+from pydoc import ModuleScanner
+
+
+def scan_modules():
+    """taken from the source code of help('modules')
+
+    https://github.com/python/cpython/blob/63298930fb531ba2bb4f23bc3b915dbf1e17e9e1/Lib/pydoc.py#L2178"""
+    modules = {}
+
+    def callback(path, modname, desc, modules=modules):
+        if modname and modname[-9:] == ".__init__":
+            modname = modname[:-9] + " (package)"
+        if modname.find(".") < 0:
+            modules[modname] = 1
+
+    def onerror(modname):
+        callback(None, modname, None)
+
+    with warnings.catch_warnings():
+        # ignore warnings from importing deprecated modules
+        warnings.simplefilter("ignore")
+        ModuleScanner().run(callback, onerror=onerror)
+    return list(modules.keys())
+
+
+def import_module(module_name):
+    import io
+    from contextlib import redirect_stdout
+    # Importing modules causes ('Constant String', 2, None, 4) and
+    # "Hello world!" to be printed to stdout.
+    f = io.StringIO()
+    with warnings.catch_warnings(), redirect_stdout(f):
+        # ignore warnings caused by importing deprecated modules
+        warnings.filterwarnings("ignore", category=DeprecationWarning)
+        try:
+            module = __import__(module_name)
+        except Exception as e:
+            return e
+    return module
+
+
+def is_child(module, item):
+    import inspect
+    item_mod = inspect.getmodule(item)
+    return item_mod is module
+
+
+def traverse(module, names, item):
+    import inspect
+    is_builtin = inspect.ismodule(item) or inspect.isbuiltin(item)
+    if is_builtin and isinstance(item.__doc__, str):
+        yield names, item.__doc__
+    attr_names = dir(item)
+    for name in attr_names:
+        if name in ['__class__', '__dict__', '__doc__', '__objclass__', '__name__', '__qualname__']:
+            continue
+        try:
+            attr = getattr(item, name)
+        except AttributeError:
+            assert name == '__abstractmethods__'
+            continue
+
+        if module is item and not is_child(module, attr):
+            continue
+
+        is_type_or_module = (type(attr) is type) or (type(attr) is type(__builtins__))
+        new_names = names.copy()
+        new_names.append(name)
+
+        if item == attr:
+            pass
+        elif not inspect.ismodule(item) and inspect.ismodule(attr):
+            pass
+        elif is_type_or_module:
+            yield from traverse(module, new_names, attr)
+        elif callable(attr) or not issubclass(type(attr), type) or type(attr).__name__ in ('getset_descriptor', 'member_descriptor'):
+            if inspect.isbuiltin(attr):
+                yield new_names, attr.__doc__
+        else:
+            assert False, (module, new_names, attr, type(attr).__name__)
+
+
+def traverse_all():
+    for module_name in scan_modules():
+        if module_name in ('this', 'antigravity'):
+            continue
+        module = import_module(module_name)
+        if hasattr(module, '__cached__'):  # python module
+            continue
+        yield from traverse(module, [module_name], module)
+
+
+def docs():
+    docs = {'.'.join(names): doc for names, doc in traverse_all()}
+    return docs
+
+if __name__ == '__main__':
+    import json
+    print(json.dumps(docs(), indent=4, sort_keys=True))

derive/src/pymodule.rs

@@ -13,7 +13,6 @@
 import re
 import sys
 import warnings
-from contextlib import redirect_stdout
 from pydoc import ModuleScanner
 
 
@@ -172,7 +171,9 @@ def onerror(modname):
     return list(modules.keys())
 
 
-def dir_of_mod_or_error(module_name, keep_other=True):
+def import_module(module_name):
+    import io
+    from contextlib import redirect_stdout
     # Importing modules causes ('Constant String', 2, None, 4) and
     # "Hello world!" to be printed to stdout.
     f = io.StringIO()
@@ -183,13 +184,23 @@ def dir_of_mod_or_error(module_name, keep_other=True):
             module = __import__(module_name)
         except Exception as e:
             return e
+    return module
+
+
+def is_child(module, item):
+    import inspect
+    item_mod = inspect.getmodule(item)
+    return item_mod is module
+
+
+def dir_of_mod_or_error(module_name, keep_other=True):
+    module = import_module(module_name)
     item_names = sorted(set(dir(module)))
     result = {}
     for item_name in item_names:
         item = getattr(module, item_name)
-        item_mod = inspect.getmodule(item)
         # don't repeat items imported from other modules
-        if keep_other or item_mod is module or item_mod is None:
+        if keep_other or is_child(module, item) or inspect.getmodule(item) is None:
             result[item_name] = extra_info(item)
     return result