From 194228850a722b01db1fea9d4402f891bf7e1cbd Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 28 Jan 2020 15:56:01 +0100 Subject: [PATCH 1/5] Python: Add tests for py/import-deprecated-module --- .../Imports/deprecated/DeprecatedModule.expected | 8 ++++++-- .../Imports/deprecated/DeprecatedModule.qlref | 2 +- .../test/query-tests/Imports/deprecated/test.py | 16 ++++++++++++---- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected index 11905b13d9cd..43cad54355b5 100644 --- a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected +++ b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected @@ -1,2 +1,6 @@ -| test.py:5:1:5:13 | Import | The rfc822 module was deprecated in version 2.3. Use email module instead. | -| test.py:6:1:6:16 | Import | The posixfile module was deprecated in version 1.5. Use email module instead. | +| test.py:2:1:2:13 | Import | The rfc822 module was deprecated in version 2.3. Use email module instead. | +| test.py:3:1:3:16 | Import | The posixfile module was deprecated in version 1.5. Use email module instead. | +| test.py:6:1:6:18 | ClassDef | The md5 module was deprecated in version 2.5. Use hashlib module instead. | +| test.py:7:5:7:18 | FunctionDef | The md5 module was deprecated in version 2.5. Use hashlib module instead. | +| test.py:8:9:8:18 | Import | The md5 module was deprecated in version 2.5. Use hashlib module instead. | +| test.py:14:5:14:23 | Import | The md5 module was deprecated in version 2.5. Use hashlib module instead. | diff --git a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.qlref b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.qlref index 3444aa9d6f69..9f87b11d807c 100644 --- a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.qlref +++ b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.qlref @@ -1 +1 @@ -Imports/DeprecatedModule.ql \ No newline at end of file +Imports/DeprecatedModule.ql diff --git a/python/ql/test/query-tests/Imports/deprecated/test.py b/python/ql/test/query-tests/Imports/deprecated/test.py index ece57bb1855c..82061bdf80cf 100644 --- a/python/ql/test/query-tests/Imports/deprecated/test.py +++ b/python/ql/test/query-tests/Imports/deprecated/test.py @@ -1,6 +1,14 @@ - - - -#Some deprecated modules +# Some deprecated modules import rfc822 import posixfile + +# TODO: We should only report a bad import once +class Foo(object): + def foo(self): + import md5 + +# TODO: Backwards compatible code, should not report +try: + from hashlib import md5 +except ImportError: + from md5 import md5 From 7949acc3ef498a0b1450d8ef6d01a8f874f4efb2 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 28 Jan 2020 15:57:44 +0100 Subject: [PATCH 2/5] Python: Autoformat --- python/ql/src/Imports/DeprecatedModule.ql | 33 ++++++++++++----------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/python/ql/src/Imports/DeprecatedModule.ql b/python/ql/src/Imports/DeprecatedModule.ql index 22f4f962e314..938bdd93c05d 100644 --- a/python/ql/src/Imports/DeprecatedModule.ql +++ b/python/ql/src/Imports/DeprecatedModule.ql @@ -12,7 +12,6 @@ import python - predicate deprecated_module(string name, string instead, int major, int minor) { name = "posixfile" and instead = "email" and major = 1 and minor = 5 or @@ -34,40 +33,44 @@ predicate deprecated_module(string name, string instead, int major, int minor) { or name = "rotor" and instead = "no replacement" and major = 2 and minor = 4 or - name = "statcache" and instead = "no replacement" and major = 2 and minor = 2 + name = "statcache" and instead = "no replacement" and major = 2 and minor = 2 or - name = "mpz" and instead = "a third party" and major = 2 and minor = 2 + name = "mpz" and instead = "a third party" and major = 2 and minor = 2 or name = "xreadlines" and instead = "no replacement" and major = 2 and minor = 3 or name = "multifile" and instead = "email" and major = 2 and minor = 5 or - name = "sets" and instead = "builtins" and major = 2 and minor = 6 + name = "sets" and instead = "builtins" and major = 2 and minor = 6 or name = "buildtools" and instead = "no replacement" and major = 2 and minor = 3 or - name = "cfmfile" and instead = "no replacement" and major = 2 and minor = 4 + name = "cfmfile" and instead = "no replacement" and major = 2 and minor = 4 or name = "macfs" and instead = "no replacement" and major = 2 and minor = 3 or - name = "md5" and instead = "hashlib" and major = 2 and minor = 5 + name = "md5" and instead = "hashlib" and major = 2 and minor = 5 or - name = "sha" and instead = "hashlib" and major = 2 and minor = 5 + name = "sha" and instead = "hashlib" and major = 2 and minor = 5 } string deprecation_message(string mod) { - exists(int major, int minor | deprecated_module(mod, _, major, minor) | - result = "The " + mod + " module was deprecated in version " + major.toString() + "." + minor.toString() + ".") + exists(int major, int minor | deprecated_module(mod, _, major, minor) | + result = "The " + mod + " module was deprecated in version " + major.toString() + "." + + minor.toString() + "." + ) } string replacement_message(string mod) { - exists(string instead | deprecated_module(mod, instead, _, _) | - result = " Use " + instead + " module instead." and not instead = "no replacement" - or - result = "" and instead = "no replacement" - ) + exists(string instead | deprecated_module(mod, instead, _, _) | + result = " Use " + instead + " module instead." and not instead = "no replacement" + or + result = "" and instead = "no replacement" + ) } from ImportExpr imp, Stmt s, Expr e -where s.getASubExpression() = e and (e = imp or e.contains(imp)) +where + s.getASubExpression() = e and + (e = imp or e.contains(imp)) select s, deprecation_message(imp.getName()) + replacement_message(imp.getName()) From e92d6c0459a71df00faffa8efb776ccc650c636c Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 28 Jan 2020 16:14:01 +0100 Subject: [PATCH 3/5] Python: Stop py/import-deprecated-module from double alerting This changes the location from the import statement, to the actual expression --- python/ql/src/Imports/DeprecatedModule.ql | 12 ++++++++---- .../Imports/deprecated/DeprecatedModule.expected | 10 ++++------ .../ql/test/query-tests/Imports/deprecated/test.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/python/ql/src/Imports/DeprecatedModule.ql b/python/ql/src/Imports/DeprecatedModule.ql index 938bdd93c05d..c3cd8d3b9510 100644 --- a/python/ql/src/Imports/DeprecatedModule.ql +++ b/python/ql/src/Imports/DeprecatedModule.ql @@ -12,6 +12,10 @@ import python +/** + * The module `name` was deprecated in Python version `major`.`minor`, + * and module `instead` should be used instead (or `instead = "no replacement"`) + */ predicate deprecated_module(string name, string instead, int major, int minor) { name = "posixfile" and instead = "email" and major = 1 and minor = 5 or @@ -69,8 +73,8 @@ string replacement_message(string mod) { ) } -from ImportExpr imp, Stmt s, Expr e +from ImportExpr imp, string name where - s.getASubExpression() = e and - (e = imp or e.contains(imp)) -select s, deprecation_message(imp.getName()) + replacement_message(imp.getName()) + name = imp.getName() and + deprecated_module(name, _, _, _) +select imp, deprecation_message(name) + replacement_message(name) diff --git a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected index 43cad54355b5..f07a22fd8a65 100644 --- a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected +++ b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected @@ -1,6 +1,4 @@ -| test.py:2:1:2:13 | Import | The rfc822 module was deprecated in version 2.3. Use email module instead. | -| test.py:3:1:3:16 | Import | The posixfile module was deprecated in version 1.5. Use email module instead. | -| test.py:6:1:6:18 | ClassDef | The md5 module was deprecated in version 2.5. Use hashlib module instead. | -| test.py:7:5:7:18 | FunctionDef | The md5 module was deprecated in version 2.5. Use hashlib module instead. | -| test.py:8:9:8:18 | Import | The md5 module was deprecated in version 2.5. Use hashlib module instead. | -| test.py:14:5:14:23 | Import | The md5 module was deprecated in version 2.5. Use hashlib module instead. | +| test.py:2:8:2:13 | ImportExpr | The rfc822 module was deprecated in version 2.3. Use email module instead. | +| test.py:3:8:3:16 | ImportExpr | The posixfile module was deprecated in version 1.5. Use email module instead. | +| test.py:8:16:8:18 | ImportExpr | The md5 module was deprecated in version 2.5. Use hashlib module instead. | +| test.py:14:10:14:12 | ImportExpr | The md5 module was deprecated in version 2.5. Use hashlib module instead. | diff --git a/python/ql/test/query-tests/Imports/deprecated/test.py b/python/ql/test/query-tests/Imports/deprecated/test.py index 82061bdf80cf..cbeaca8d7e4f 100644 --- a/python/ql/test/query-tests/Imports/deprecated/test.py +++ b/python/ql/test/query-tests/Imports/deprecated/test.py @@ -2,7 +2,7 @@ import rfc822 import posixfile -# TODO: We should only report a bad import once +# We should only report a bad import once class Foo(object): def foo(self): import md5 From 6c7cddf258f3a061ad928754e5368f845875303d Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 28 Jan 2020 16:36:47 +0100 Subject: [PATCH 4/5] Python: py/import-deprecated-module handle backwards compatible code --- python/ql/src/Imports/DeprecatedModule.ql | 9 +++++++-- python/ql/src/semmle/python/objects/ObjectAPI.qll | 5 +++++ .../Imports/deprecated/DeprecatedModule.expected | 1 - python/ql/test/query-tests/Imports/deprecated/test.py | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/ql/src/Imports/DeprecatedModule.ql b/python/ql/src/Imports/DeprecatedModule.ql index c3cd8d3b9510..baea7a137d23 100644 --- a/python/ql/src/Imports/DeprecatedModule.ql +++ b/python/ql/src/Imports/DeprecatedModule.ql @@ -73,8 +73,13 @@ string replacement_message(string mod) { ) } -from ImportExpr imp, string name +from ImportExpr imp, string name, string instead where name = imp.getName() and - deprecated_module(name, _, _, _) + deprecated_module(name, instead, _, _) and + not exists(Try try, ExceptStmt except | except = try.getAHandler() + | + except.getType().pointsTo(ClassValue::importError()) and + except.containsInScope(imp) + ) select imp, deprecation_message(name) + replacement_message(name) diff --git a/python/ql/src/semmle/python/objects/ObjectAPI.qll b/python/ql/src/semmle/python/objects/ObjectAPI.qll index 7884daa7b069..c9dcb8dd658d 100644 --- a/python/ql/src/semmle/python/objects/ObjectAPI.qll +++ b/python/ql/src/semmle/python/objects/ObjectAPI.qll @@ -714,4 +714,9 @@ module ClassValue { result = TBuiltinClassObject(Builtin::builtin("NameError")) } + /** Get the `ClassValue` for the `ImportError` class. */ + ClassValue importError() { + result = TBuiltinClassObject(Builtin::builtin("ImportError")) + } + } diff --git a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected index f07a22fd8a65..1425c18447c6 100644 --- a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected +++ b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected @@ -1,4 +1,3 @@ | test.py:2:8:2:13 | ImportExpr | The rfc822 module was deprecated in version 2.3. Use email module instead. | | test.py:3:8:3:16 | ImportExpr | The posixfile module was deprecated in version 1.5. Use email module instead. | | test.py:8:16:8:18 | ImportExpr | The md5 module was deprecated in version 2.5. Use hashlib module instead. | -| test.py:14:10:14:12 | ImportExpr | The md5 module was deprecated in version 2.5. Use hashlib module instead. | diff --git a/python/ql/test/query-tests/Imports/deprecated/test.py b/python/ql/test/query-tests/Imports/deprecated/test.py index cbeaca8d7e4f..ce70d29794eb 100644 --- a/python/ql/test/query-tests/Imports/deprecated/test.py +++ b/python/ql/test/query-tests/Imports/deprecated/test.py @@ -7,7 +7,7 @@ class Foo(object): def foo(self): import md5 -# TODO: Backwards compatible code, should not report +# Backwards compatible code, should not report try: from hashlib import md5 except ImportError: From 4ca72de4cdf9068ff4973862967ff641c9385e62 Mon Sep 17 00:00:00 2001 From: Rasmus Wriedt Larsen Date: Tue, 28 Jan 2020 16:43:45 +0100 Subject: [PATCH 5/5] Python: Fix recommended module for deprecated posixfile $ python2 -W default -c 'import posixfile' -c:1: DeprecationWarning: The posixfile module is deprecated; fcntl.lockf() provides better locking https://docs.python.org/2.7/library/posixfile.html --- python/ql/src/Imports/DeprecatedModule.ql | 2 +- .../query-tests/Imports/deprecated/DeprecatedModule.expected | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ql/src/Imports/DeprecatedModule.ql b/python/ql/src/Imports/DeprecatedModule.ql index baea7a137d23..5ecd7f45cfe4 100644 --- a/python/ql/src/Imports/DeprecatedModule.ql +++ b/python/ql/src/Imports/DeprecatedModule.ql @@ -17,7 +17,7 @@ import python * and module `instead` should be used instead (or `instead = "no replacement"`) */ predicate deprecated_module(string name, string instead, int major, int minor) { - name = "posixfile" and instead = "email" and major = 1 and minor = 5 + name = "posixfile" and instead = "fcntl" and major = 1 and minor = 5 or name = "gopherlib" and instead = "no replacement" and major = 2 and minor = 5 or diff --git a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected index 1425c18447c6..d379a896f20a 100644 --- a/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected +++ b/python/ql/test/query-tests/Imports/deprecated/DeprecatedModule.expected @@ -1,3 +1,3 @@ | test.py:2:8:2:13 | ImportExpr | The rfc822 module was deprecated in version 2.3. Use email module instead. | -| test.py:3:8:3:16 | ImportExpr | The posixfile module was deprecated in version 1.5. Use email module instead. | +| test.py:3:8:3:16 | ImportExpr | The posixfile module was deprecated in version 1.5. Use fcntl module instead. | | test.py:8:16:8:18 | ImportExpr | The md5 module was deprecated in version 2.5. Use hashlib module instead. |