Implement PEP 3121: new module initialization and finalization API.

Author: loewis

Doc/c-api/import.rst

@@ -200,7 +200,7 @@ Importing Modules
    tricks with this to provide a dynamically created collection of frozen modules.
 
 
-.. cfunction:: int PyImport_AppendInittab(char *name, void (*initfunc)(void))
+.. cfunction:: int PyImport_AppendInittab(char *name, PyObject* (*initfunc)(void))
 
    Add a single module to the existing table of built-in modules.  This is a
    convenience wrapper around :cfunc:`PyImport_ExtendInittab`, returning ``-1`` if
@@ -221,7 +221,7 @@ Importing Modules
 
       struct _inittab {
           char *name;
-          void (*initfunc)(void);
+          PyObject* (*initfunc)(void);
       };
 
 

Doc/extending/extending.rst

@@ -191,21 +191,22 @@ usually declare a static object variable at the beginning of your file::
 
    static PyObject *SpamError;
 
-and initialize it in your module's initialization function (:cfunc:`initspam`)
+and initialize it in your module's initialization function (:cfunc:`PyInit_spam`)
 with an exception object (leaving out the error checking for now)::
 
    PyMODINIT_FUNC
-   initspam(void)
+   PyInit_spam(void)
    {
        PyObject *m;
 
-       m = Py_InitModule("spam", SpamMethods);
+       m = PyModule_Create(&spammodule);
        if (m == NULL)
-           return;
+           return NULL;
 
        SpamError = PyErr_NewException("spam.error", NULL, NULL);
        Py_INCREF(SpamError);
        PyModule_AddObject(m, "error", SpamError);
+       return m;
    }
 
 Note that the Python name for the exception object is :exc:`spam.error`.  The
@@ -303,49 +304,64 @@ accept a third ``PyObject *`` parameter which will be a dictionary of keywords.
 Use :cfunc:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a
 function.
 
-The method table must be passed to the interpreter in the module's
+The method table must be referenced in the module definition structure::
+
+   struct PyModuleDef spammodule = {
+      PyModuleDef_HEAD_INIT,
+      "spam",   /* name of module */
+      spam_doc, /* module documentation, may be NULL */
+      -1,       /* size of per-interpreter state of the module,
+                   or -1 if the module keeps state in global variables. */
+      SpamMethods
+   };
+
+This structure, in turn, must be passed to the interpreter in the module's
 initialization function.  The initialization function must be named
-:cfunc:`initname`, where *name* is the name of the module, and should be the
+:cfunc:`PyInit_name`, where *name* is the name of the module, and should be the
 only non-\ ``static`` item defined in the module file::
 
    PyMODINIT_FUNC
-   initspam(void)
+   PyInit_spam(void)
    {
-       (void) Py_InitModule("spam", SpamMethods);
+       return PyModule_Create(&spammodule);
    }
 
 Note that PyMODINIT_FUNC declares the function as ``void`` return type,
 declares any special linkage declarations required by the platform, and for  C++
 declares the function as ``extern "C"``.
 
 When the Python program imports module :mod:`spam` for the first time,
-:cfunc:`initspam` is called. (See below for comments about embedding Python.)
-It calls :cfunc:`Py_InitModule`, which creates a "module object" (which is
-inserted in the dictionary ``sys.modules`` under the key ``"spam"``), and
+:cfunc:`PyInit_spam` is called. (See below for comments about embedding Python.)
+It calls :cfunc:`PyModule_Create`, which returns a module object, and
 inserts built-in function objects into the newly created module based upon the
-table (an array of :ctype:`PyMethodDef` structures) that was passed as its
-second argument. :cfunc:`Py_InitModule` returns a pointer to the module object
-that it creates (which is unused here).  It may abort with a fatal error for
+table (an array of :ctype:`PyMethodDef` structures) found in the module definition. 
+:cfunc:`PyModule_Create` returns a pointer to the module object
+that it creates.  It may abort with a fatal error for
 certain errors, or return *NULL* if the module could not be initialized
-satisfactorily.
+satisfactorily. The init function must return the module object to its caller,
+so that it then gets inserted into ``sys.modules``.
 
-When embedding Python, the :cfunc:`initspam` function is not called
+When embedding Python, the :cfunc:`PyInit_spam` function is not called
 automatically unless there's an entry in the :cdata:`_PyImport_Inittab` table.
-The easiest way to handle this is to statically initialize your
-statically-linked modules by directly calling :cfunc:`initspam` after the call
-to :cfunc:`Py_Initialize`::
+To add the module to the initialization table, use :cfunc:`PyImport_AppendInittab`,
+optionally followed by an import of the module::
 
    int
    main(int argc, char *argv[])
    {
+       /* Add a builtin module, before Py_Initialize */
+       PyImport_AppendInittab("spam", PyInit_spam);
+
        /* Pass argv[0] to the Python interpreter */
        Py_SetProgramName(argv[0]);
 
        /* Initialize the Python interpreter.  Required. */
        Py_Initialize();
 
-       /* Add a static module */
-       initspam();
+       /* Optionally import the module; alternatively,
+          import can be deferred until the embedded script
+          imports it. */
+       PyImport_ImportModule("spam");
 
 An example may be found in the file :file:`Demo/embed/demo.c` in the Python
 source distribution.
@@ -1154,15 +1170,15 @@ exporting module, not a client module. Finally, the module's initialization
 function must take care of initializing the C API pointer array::
 
    PyMODINIT_FUNC
-   initspam(void)
+   PyInit_spam(void)
    {
        PyObject *m;
        static void *PySpam_API[PySpam_API_pointers];
        PyObject *c_api_object;
 
-       m = Py_InitModule("spam", SpamMethods);
+       m = PyModule_Create(&spammodule);
        if (m == NULL)
-           return;
+           return NULL;
 
        /* Initialize the C API pointer array */
        PySpam_API[PySpam_System_NUM] = (void *)PySpam_System;
@@ -1172,10 +1188,11 @@ function must take care of initializing the C API pointer array::
 
        if (c_api_object != NULL)
            PyModule_AddObject(m, "_C_API", c_api_object);
+       return m;
    }
 
 Note that ``PySpam_API`` is declared ``static``; otherwise the pointer
-array would disappear when :func:`initspam` terminates!
+array would disappear when :func:`PyInit_spam` terminates!
 
 The bulk of the work is in the header file :file:`spammodule.h`, which looks
 like this::

Include/import.h

@@ -33,17 +33,17 @@ PyAPI_FUNC(int) _PyImport_IsScript(struct filedescr *);
 PyAPI_FUNC(void) _PyImport_ReInitLock(void);
 
 PyAPI_FUNC(PyObject *)_PyImport_FindExtension(char *, char *);
-PyAPI_FUNC(PyObject *)_PyImport_FixupExtension(char *, char *);
+PyAPI_FUNC(int)_PyImport_FixupExtension(PyObject*, char *, char *);
 
 struct _inittab {
     char *name;
-    void (*initfunc)(void);
+    PyObject* (*initfunc)(void);
 };
 
 PyAPI_DATA(PyTypeObject) PyNullImporter_Type;
 PyAPI_DATA(struct _inittab *) PyImport_Inittab;
 
-PyAPI_FUNC(int) PyImport_AppendInittab(char *name, void (*initfunc)(void));
+PyAPI_FUNC(int) PyImport_AppendInittab(char *name, PyObject* (*initfunc)(void));
 PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab);
 
 struct _frozen {

Include/modsupport.h

@@ -89,42 +89,18 @@ PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char
    9-Jan-1995	GvR	Initial version (incompatible with older API)
 */
 
-#ifdef MS_WINDOWS
-/* Special defines for Windows versions used to live here.  Things
-   have changed, and the "Version" is now in a global string variable.
-   Reason for this is that this for easier branding of a "custom DLL"
-   without actually needing a recompile.  */
-#endif /* MS_WINDOWS */
-
-#if SIZEOF_SIZE_T != SIZEOF_INT
-/* On a 64-bit system, rename the Py_InitModule4 so that 2.4
-   modules cannot get loaded into a 2.5 interpreter */
-#define Py_InitModule4 Py_InitModule4_64
-#endif
-
 #ifdef Py_TRACE_REFS
- /* When we are tracing reference counts, rename Py_InitModule4 so
+ /* When we are tracing reference counts, rename PyModule_New2 so
     modules compiled with incompatible settings will generate a
     link-time error. */
- #if SIZEOF_SIZE_T != SIZEOF_INT
- #undef Py_InitModule4
- #define Py_InitModule4 Py_InitModule4TraceRefs_64
- #else
- #define Py_InitModule4 Py_InitModule4TraceRefs
- #endif
+ #define PyModule_New2 PyModule_Create2TraceRefs
 #endif
 
-PyAPI_FUNC(PyObject *) Py_InitModule4(const char *name, PyMethodDef *methods,
-                                      const char *doc, PyObject *self,
-                                      int apiver);
-
-#define Py_InitModule(name, methods) \
-	Py_InitModule4(name, methods, (char *)NULL, (PyObject *)NULL, \
-		       PYTHON_API_VERSION)
+PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
+                                     int apiver);
 
-#define Py_InitModule3(name, methods, doc) \
-	Py_InitModule4(name, methods, doc, (PyObject *)NULL, \
-		       PYTHON_API_VERSION)
+#define PyModule_Create(module) \
+	PyModule_Create2(module, PYTHON_API_VERSION)
 
 PyAPI_DATA(char *) _Py_PackageContext;
 

Include/moduleobject.h

@@ -17,6 +17,30 @@ PyAPI_FUNC(PyObject *) PyModule_GetDict(PyObject *);
 PyAPI_FUNC(const char *) PyModule_GetName(PyObject *);
 PyAPI_FUNC(const char *) PyModule_GetFilename(PyObject *);
 PyAPI_FUNC(void) _PyModule_Clear(PyObject *);
+PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);
+PyAPI_FUNC(void*) PyModule_GetState(PyObject*);
+
+typedef struct PyModuleDef_Base {
+  PyObject_HEAD
+  PyObject* (*m_init)(void);
+  Py_ssize_t m_index;
+  PyObject* m_copy;
+} PyModuleDef_Base;
+
+#define PyModuleDef_HEAD_INIT {PyObject_HEAD_INIT(NULL)}
+
+typedef struct PyModuleDef{
+  PyModuleDef_Base m_base;
+  const char* m_name;
+  const char* m_doc;
+  Py_ssize_t m_size;
+  PyMethodDef *m_methods;
+  inquiry m_reload;
+  traverseproc m_traverse;
+  inquiry m_clear;
+  freefunc m_free;
+}PyModuleDef;
+
 
 #ifdef __cplusplus
 }

Include/pyport.h

@@ -511,9 +511,9 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 			/* module init functions inside the core need no external linkage */
 			/* except for Cygwin to handle embedding */
 #			if defined(__CYGWIN__)
-#				define PyMODINIT_FUNC __declspec(dllexport) void
+#				define PyMODINIT_FUNC __declspec(dllexport) PyObject*
 #			else /* __CYGWIN__ */
-#				define PyMODINIT_FUNC void
+#				define PyMODINIT_FUNC PyObject*
 #			endif /* __CYGWIN__ */
 #		else /* Py_BUILD_CORE */
 			/* Building an extension module, or an embedded situation */
@@ -526,9 +526,9 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 #			define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE
 			/* module init functions outside the core must be exported */
 #			if defined(__cplusplus)
-#				define PyMODINIT_FUNC extern "C" __declspec(dllexport) void
+#				define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject*
 #			else /* __cplusplus */
-#				define PyMODINIT_FUNC __declspec(dllexport) void
+#				define PyMODINIT_FUNC __declspec(dllexport) PyObject*
 #			endif /* __cplusplus */
 #		endif /* Py_BUILD_CORE */
 #	endif /* HAVE_DECLSPEC */
@@ -543,9 +543,9 @@ extern pid_t forkpty(int *, char *, struct termios *, struct winsize *);
 #endif
 #ifndef PyMODINIT_FUNC
 #	if defined(__cplusplus)
-#		define PyMODINIT_FUNC extern "C" void
+#		define PyMODINIT_FUNC extern "C" PyObject*
 #	else /* __cplusplus */
-#		define PyMODINIT_FUNC void
+#		define PyMODINIT_FUNC PyObject*
 #	endif /* __cplusplus */
 #endif
 

Include/pystate.h

@@ -19,6 +19,7 @@ typedef struct _is {
     struct _ts *tstate_head;
 
     PyObject *modules;
+    PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
     PyObject *modules_reloading;
@@ -107,6 +108,8 @@ typedef struct _ts {
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_New(void);
 PyAPI_FUNC(void) PyInterpreterState_Clear(PyInterpreterState *);
 PyAPI_FUNC(void) PyInterpreterState_Delete(PyInterpreterState *);
+PyAPI_FUNC(int) _PyState_AddModule(PyObject*, struct PyModuleDef*);
+PyAPI_FUNC(PyObject*) PyState_FindModule(struct PyModuleDef*);
 
 PyAPI_FUNC(PyThreadState *) PyThreadState_New(PyInterpreterState *);
 PyAPI_FUNC(void) PyThreadState_Clear(PyThreadState *);

Include/warnings.h

@@ -4,7 +4,7 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(void) _PyWarnings_Init(void);
+PyAPI_FUNC(PyObject*) _PyWarnings_Init(void);
 
 PyAPI_FUNC(int) PyErr_WarnEx(PyObject *, const char *, Py_ssize_t);
 PyAPI_FUNC(int) PyErr_WarnExplicit(PyObject *, const char *, const char *, int,

Lib/test/test_sys.py

@@ -458,7 +458,7 @@ def get_gen(): yield 1
         # builtin_function_or_method
         self.check_sizeof(abs, h + 3*p)
         # module
-        self.check_sizeof(unittest, h + p)
+        self.check_sizeof(unittest, h + 3*p)
         # range
         self.check_sizeof(range(1), h + 3*p)
         # slice

Misc/NEWS

@@ -12,6 +12,8 @@ What's new in Python 3.0b1?
 Core and Builtins
 -----------------
 
+- Implement PEP 3121: new module initialization and finalization API.
+
 - Removed the already-defunct ``-t`` option.
 
 - Issue #2957: Corrected a ValueError "recursion limit exceeded", when