From f274b325fac8e47df6d45260111d6517abddde7e Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 28 Jul 2019 17:51:37 +0100 Subject: [PATCH 001/309] Add missing methods to WrapFS (#321) * fix for makedirs race condition * changelog * add missing methods to wrapfs * added changelog * fix removetree --- CHANGELOG.md | 1 + fs/wrapfs.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2be6711..30c9fe72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Restored fs.path import - Fixed potential race condition in makedirs. Fixes [#310](https://github.com/PyFilesystem/pyfilesystem2/issues/310) +- Added missing methods to WrapFS. Fixed [#294](https://github.com/PyFilesystem/pyfilesystem2/issues/294) ### Changed diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 5ff7d3d2..34e83fae 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -10,9 +10,9 @@ from . import errors from .base import FS -from .copy import copy_file +from .copy import copy_file, copy_dir from .info import Info -from .move import move_file +from .move import move_file, move_dir from .path import abspath, normpath from .error_tools import unwrap_errors @@ -180,6 +180,17 @@ def move(self, src_path, dst_path, overwrite=False): raise errors.DestinationExists(_dst_path) move_file(src_fs, _src_path, dst_fs, _dst_path) + def copydir(self, src_path, dst_path, create=False): + # type: (Text, Text, bool) -> None + src_fs, _src_path = self.delegate_path(src_path) + dst_fs, _dst_path = self.delegate_path(dst_path) + with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): + if not create and not dst_fs.exists(_dst_path): + raise errors.ResourceNotFound(dst_path) + if not src_fs.getinfo(_src_path).is_dir: + raise errors.DirectoryExpected(src_path) + move_dir(src_fs, _src_path, dst_fs, _dst_path) + def openbin(self, path, mode="r", buffering=-1, **options): # type: (Text, Text, int, **Any) -> BinaryIO self.check() @@ -205,6 +216,16 @@ def removedir(self, path): with unwrap_errors(path): _fs.removedir(_path) + def removetree(self, dir_path): + # type: (Text) -> None + self.check() + _path = abspath(normpath(dir_path)) + if _path == "/": + raise errors.RemoveRootError() + _fs, _path = self.delegate_path(dir_path) + with unwrap_errors(dir_path): + _fs.removetree(_path) + def scandir( self, path, # type: Text @@ -247,6 +268,17 @@ def copy(self, src_path, dst_path, overwrite=False): raise errors.DestinationExists(_dst_path) copy_file(src_fs, _src_path, dst_fs, _dst_path) + def copydir(self, src_path, dst_path, create=False): + # type: (Text, Text, bool) -> None + src_fs, _src_path = self.delegate_path(src_path) + dst_fs, _dst_path = self.delegate_path(dst_path) + with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): + if not create and not dst_fs.exists(_dst_path): + raise errors.ResourceNotFound(dst_path) + if not src_fs.getinfo(_src_path).is_dir: + raise errors.DirectoryExpected(src_path) + copy_dir(src_fs, _src_path, dst_fs, _dst_path) + def create(self, path, wipe=False): # type: (Text, bool) -> bool self.check() @@ -262,6 +294,13 @@ def desc(self, path): desc = _fs.desc(_path) return desc + def download(self, path, file, chunk_size=None, **options): + # type: (Text, BinaryIO, Optional[int], **Any) -> None + self.check() + _fs, _path = self.delegate_path(path) + with unwrap_errors(path): + _fs.download(_path, file, chunk_size=chunk_size, **options) + def exists(self, path): # type: (Text) -> bool self.check() From d6d51f2ba12a4b5e9137fcc892c4c17fe83279d0 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 28 Jul 2019 17:52:37 +0100 Subject: [PATCH 002/309] version bump --- CHANGELOG.md | 2 +- fs/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30c9fe72..fd41e91c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.9] - (Unreleased) +## [2.4.9] - 2019-07-28 ### Fixed diff --git a/fs/_version.py b/fs/_version.py index 2d80d12d..e8e9d215 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.8" +__version__ = "2.4.9" From 55dfeb36a862f0c490b1c8974fa544300d615850 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Mon, 29 Jul 2019 22:20:24 +0100 Subject: [PATCH 003/309] fix movedir (#323) --- CHANGELOG.md | 6 ++++++ fs/wrapfs.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd41e91c..e23c2eaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.4.10] - 2019-07-29 + +### Fixed + +- Fixed broken WrapFS.movedir [#322](https://github.com/PyFilesystem/pyfilesystem2/issues/322) + ## [2.4.9] - 2019-07-28 ### Fixed diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 34e83fae..6565a4ce 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -180,7 +180,7 @@ def move(self, src_path, dst_path, overwrite=False): raise errors.DestinationExists(_dst_path) move_file(src_fs, _src_path, dst_fs, _dst_path) - def copydir(self, src_path, dst_path, create=False): + def movedir(self, src_path, dst_path, create=False): # type: (Text, Text, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) From a57ef8b799c3c63dee43e5f3528de77ba9a9cb83 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Mon, 29 Jul 2019 22:21:18 +0100 Subject: [PATCH 004/309] version bump --- fs/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/_version.py b/fs/_version.py index e8e9d215..f65d518f 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.9" +__version__ = "2.4.10" From 108c35a9210777ef732569902cc5c28872804f90 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Tue, 30 Jul 2019 13:21:20 +0100 Subject: [PATCH 005/309] link to ResourceType in docs --- docs/source/info.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/info.rst b/docs/source/info.rst index 51719dbf..ba0d6d12 100644 --- a/docs/source/info.rst +++ b/docs/source/info.rst @@ -88,13 +88,13 @@ size int Number of bytes used to store the the overhead (in bytes) used to store the directory entry. type ResourceType Resource type, one of the values - defined in :class:`~fs.ResourceType`. + defined in :class:`~fs.enums.ResourceType`. ================ =================== ========================================== The time values (``accessed_time``, ``created_time`` etc.) may be ``None`` if the filesystem doesn't store that information. The ``size`` and ``type`` keys are guaranteed to be available, although ``type`` may -be :attr:`~fs.ResourceType.unknown` if the filesystem is unable to +be :attr:`~fs.enums.ResourceType.unknown` if the filesystem is unable to retrieve the resource type. Access Namespace From 1bcedffd88476ad83c57d236aefbb34337443d74 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Tue, 30 Jul 2019 13:23:04 +0100 Subject: [PATCH 006/309] ResourceType links --- fs/base.py | 6 +++--- fs/info.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fs/base.py b/fs/base.py index 297d1587..18f3ccdd 100644 --- a/fs/base.py +++ b/fs/base.py @@ -175,7 +175,7 @@ def listdir(self, path): This method will return a list of the resources in a directory. A *resource* is a file, directory, or one of the other types - defined in `~fs.ResourceType`. + defined in `~fs.enums.ResourceType`. Arguments: path (str): A path to a directory on the filesystem @@ -823,11 +823,11 @@ def gettype(self, path): path (str): A path on the filesystem. Returns: - ~fs.ResourceType: the type of the resource. + ~fs.enums.ResourceType: the type of the resource. A type of a resource is an integer that identifies the what the resource references. The standard type integers may be one - of the values in the `~fs.ResourceType` enumerations. + of the values in the `~fs.enums.ResourceType` enumerations. The most common resource types, supported by virtually all filesystems are ``directory`` (1) and ``file`` (2), but the diff --git a/fs/info.py b/fs/info.py index d01c4c44..194dd48f 100644 --- a/fs/info.py +++ b/fs/info.py @@ -260,7 +260,7 @@ def is_link(self): @property def type(self): # type: () -> ResourceType - """`~fs.ResourceType`: the type of the resource. + """`~fs.enums.ResourceType`: the type of the resource. Requires the ``"details"`` namespace. From 53a7b3842b0b19e835622334e62082d5c6915ae7 Mon Sep 17 00:00:00 2001 From: Andrew Scheller Date: Sat, 3 Aug 2019 16:49:36 +0100 Subject: [PATCH 007/309] Add AppVeyor badge to README (#328) Fixes #325 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 27f02063..8616bc3b 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Python's Filesystem abstraction layer. [![PyPI version](https://badge.fury.io/py/fs.svg)](https://badge.fury.io/py/fs) [![PyPI](https://img.shields.io/pypi/pyversions/fs.svg)](https://pypi.org/project/fs/) [![Build Status](https://travis-ci.org/PyFilesystem/pyfilesystem2.svg?branch=master)](https://travis-ci.org/PyFilesystem/pyfilesystem2) +[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/pyfilesystem/pyfilesystem2?branch=master&svg=true)](https://ci.appveyor.com/project/willmcgugan/pyfilesystem2) [![Coverage Status](https://coveralls.io/repos/github/PyFilesystem/pyfilesystem2/badge.svg)](https://coveralls.io/github/PyFilesystem/pyfilesystem2) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/30ad6445427349218425d93886ade9ee)](https://www.codacy.com/app/will-mcgugan/pyfilesystem2?utm_source=github.com&utm_medium=referral&utm_content=PyFilesystem/pyfilesystem2&utm_campaign=Badge_Grade) [![Code Health](https://landscape.io/github/PyFilesystem/pyfilesystem2/master/landscape.svg?style=flat)](https://landscape.io/github/PyFilesystem/pyfilesystem2/master) From fdf85943d6cfeffb7f5f0fd6257dfe86e659ac9e Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 4 Aug 2019 15:30:39 +0100 Subject: [PATCH 008/309] fix path test --- tests/test_imports.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_imports.py b/tests/test_imports.py index 72fa6fba..8d8af34a 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -6,6 +6,7 @@ class TestImports(unittest.TestCase): def test_import_path(self): """Test import fs also imports other symbols.""" restore_fs = sys.modules.pop("fs") + sys.modules.pop("fs.path") try: import fs From c6b71b712f17aacd477bac9deb1fcf913f13d831 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 4 Aug 2019 15:50:52 +0100 Subject: [PATCH 009/309] remove temporary directories during tests --- fs/tempfs.py | 3 ++- tests/test_osfs.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fs/tempfs.py b/fs/tempfs.py index 04af76f9..293a692e 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -86,6 +86,7 @@ def clean(self): except Exception as error: if not self._ignore_clean_errors: raise errors.OperationFailed( - msg="failed to remove temporary directory", exc=error + msg="failed to remove temporary directory; {}".format(error), + exc=error, ) self._cleaned = True diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 601e0b45..8ed361fb 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -28,6 +28,11 @@ def make_fs(self): def destroy_fs(self, fs): self.fs.close() + try: + shutil.rmtree(fs.getsyspath("/")) + except OSError: + # Already deleted + pass def _get_real_path(self, path): _path = os.path.join(self.fs.root_path, relpath(path)) From 7d7539a2766217902e8de0002b10846fc4c21d9e Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 4 Aug 2019 16:11:47 +0100 Subject: [PATCH 010/309] remove tar tmp files --- tests/test_tarfs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index 610c7a31..ce8794cd 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -60,6 +60,7 @@ def make_fs(self): def destroy_fs(self, fs): fs.close() + os.remove(fs._tar_file) del fs._tar_file @@ -92,6 +93,7 @@ def make_fs(self): def destroy_fs(self, fs): fs.close() + os.remove(fs._tar_file) del fs._tar_file def assert_is_bzip(self): From 84719bebdee4b516bf4eef2a0dd2ac26339ab59b Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 4 Aug 2019 20:17:53 +0100 Subject: [PATCH 011/309] typing fixes (#334) * typing fixes * changelog * install mypy in travis * removed mypy * add typecheck to travis * second attempt * add typecheck to matrix * safer close * ignore type error * handle double close --- .travis.yml | 6 ++++++ CHANGELOG.md | 8 ++++++++ fs/_fscompat.py | 2 +- fs/_version.py | 2 +- fs/glob.py | 13 +++++++------ fs/memoryfs.py | 3 ++- fs/mode.py | 1 + fs/opener/registry.py | 4 ++-- fs/osfs.py | 14 ++++++-------- 9 files changed, 34 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index ad36cf4e..965e485f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,13 @@ matrix: - SETUPTOOLS=setuptools PIP=pip dist: xenial sudo: true + install: + - pip install mypy + - make typecheck - python: "3.6" + install: + - pip install mypy + - make typecheck env: - SETUPTOOLS=setuptools PIP=pip - python: "3.5" diff --git a/CHANGELOG.md b/CHANGELOG.md index e23c2eaf..f593b91f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.4.11] - Unreleased + +### Fixed + +- Fixed tests leaving tmp files +- Fixed typing issues +- Fixed link namespace returning bytes + ## [2.4.10] - 2019-07-29 ### Fixed diff --git a/fs/_fscompat.py b/fs/_fscompat.py index 54717b7f..ca7f5431 100644 --- a/fs/_fscompat.py +++ b/fs/_fscompat.py @@ -11,7 +11,7 @@ from os import fspath except ImportError: - def fspath(path): + def fspath(path): # type: ignore """Return the path representation of a path-like object. If str or bytes is passed in, it is returned unchanged. Otherwise the diff --git a/fs/_version.py b/fs/_version.py index f65d518f..207d7928 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.10" +__version__ = "2.4.11a0" diff --git a/fs/glob.py b/fs/glob.py index 09927952..bff4790f 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -10,20 +10,21 @@ from . import wildcard -_PATTERN_CACHE = LRUCache( - 1000 -) # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]] - -GlobMatch = namedtuple('GlobMatch', ["path", "info"]) +GlobMatch = namedtuple("GlobMatch", ["path", "info"]) Counts = namedtuple("Counts", ["files", "directories", "data"]) LineCounts = namedtuple("LineCounts", ["lines", "non_blank"]) if False: # typing.TYPE_CHECKING - from typing import Iterator, List, Optional, Tuple + from typing import Iterator, List, Optional, Pattern, Text, Tuple from .base import FS from .info import Info +_PATTERN_CACHE = LRUCache( + 1000 +) # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]] + + def _translate_glob(pattern, case_sensitive=True): levels = 0 recursive = False diff --git a/fs/memoryfs.py b/fs/memoryfs.py index dcee49db..dcc03050 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -342,7 +342,8 @@ def _get_dir_entry(self, dir_path): def close(self): # type: () -> None - self.root = None + if not self._closed: + del self.root return super(MemoryFS, self).close() def getinfo(self, path, namespaces=None): diff --git a/fs/mode.py b/fs/mode.py index 96e00566..76b665d6 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -67,6 +67,7 @@ def __contains__(self, character): # type: (object) -> bool """Check if a mode contains a given character. """ + assert isinstance(character, Text) return character in self._mode def to_platform(self): diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 80183295..0ed74339 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -57,7 +57,7 @@ def __repr__(self): return "".format(self.protocols) def install(self, opener): - # type: (Union[Type[Opener], Opener, Callable[[], Opener]]) -> None + # type: (Union[Type[Opener], Opener, Callable[[], Opener]]) -> Opener """Install an opener. Arguments: @@ -76,7 +76,7 @@ class ArchiveOpener(Opener): assert _opener.protocols, "must list one or more protocols" for protocol in _opener.protocols: self._protocols[protocol] = _opener - return opener + return _opener @property def protocols(self): diff --git a/fs/osfs.py b/fs/osfs.py index b891380e..10f8713a 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -28,7 +28,7 @@ try: from scandir import scandir # type: ignore except ImportError: # pragma: no cover - scandir = None # pragma: no cover + scandir = None # type: ignore # pragma: no cover try: from os import sendfile @@ -36,7 +36,7 @@ try: from sendfile import sendfile # type: ignore except ImportError: - sendfile = None # pragma: no cover + sendfile = None # type: ignore # pragma: no cover from . import errors from .errors import FileExists @@ -186,7 +186,7 @@ def __str__(self): return fmt.format(_class_name.lower(), self.root_path) def _to_sys_path(self, path): - # type: (Text) -> Text + # type: (Text) -> bytes """Convert a FS path to a path on the OS. """ sys_path = fsencode( @@ -266,13 +266,11 @@ def _gettarget(self, sys_path): if hasattr(os, "readlink"): try: if _WINDOWS_PLATFORM: # pragma: no cover - target = os.readlink(sys_path) + return os.readlink(sys_path) else: - target = os.readlink(fsencode(sys_path)) + return fsdecode(os.readlink(fsencode(sys_path))) except OSError: pass - else: - return target return None def _make_link_info(self, sys_path): @@ -484,7 +482,7 @@ def _scandir(self, path, namespaces=None): self._root_path, path.lstrip("/").replace("/", os.sep) ) else: - sys_path = self._to_sys_path(_path) + sys_path = self._to_sys_path(_path) # type: ignore with convert_os_errors("scandir", path, directory=True): for dir_entry in scandir(sys_path): info = { From 667d47753cbb282ac4c46d566324a895fc787e63 Mon Sep 17 00:00:00 2001 From: Diego Argueta Date: Fri, 23 Aug 2019 09:19:08 -0700 Subject: [PATCH 012/309] Migrate tests to Pytest (#337) * Remove unnecessary requirements.txt * Make complete changeover to pytest * More pytest porting * Fix Appveyor, missing typechecking * Get rid of collections import warning * Mock out AppFS user directories * Remove confusing stack trace change via six * Update changelog * Forgot create=True in a test * Document test changes in Implementing * PR feedback * Add `slow` markers back in * Accumulate coverage across versions * Update changelog * Remove unused import. * Enable branch coverage --- .travis.yml | 42 ++++++++++++++---------------------- CHANGELOG.md | 9 ++++++++ appveyor.yml | 4 ++-- docs/source/implementers.rst | 7 ++++++ fs/error_tools.py | 3 +-- fs/test.py | 19 ++++++++++------ fs/walk.py | 4 +--- requirements.txt | 7 ------ setup.cfg | 6 ++++++ setup.py | 2 -- testrequirements.txt | 18 ++++++++++------ tests/conftest.py | 34 +++++++++++++++++++++++++++++ tests/test_appfs.py | 42 +++++++++++++++++------------------- tests/test_base.py | 6 ------ tests/test_encoding.py | 6 +++++- tests/test_ftp_parse.py | 6 +++++- tests/test_ftpfs.py | 8 +++---- tests/test_memoryfs.py | 6 ++++-- tests/test_opener.py | 11 +++++++--- tests/test_osfs.py | 13 +++++++---- tests/test_tarfs.py | 9 +++----- tests/test_tempfs.py | 6 +++++- tox.ini | 25 ++++++++++----------- 23 files changed, 172 insertions(+), 121 deletions(-) delete mode 100644 requirements.txt create mode 100644 tests/conftest.py diff --git a/.travis.yml b/.travis.yml index 965e485f..d435957b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,34 +1,25 @@ +dist: xenial sudo: false language: python +python: + - '2.7' + - '3.4' + - '3.5' + - '3.6' + - '3.7' +# TODO (dargueta): Enable these once we figure out FTP timing issues in PyPy tests +# - 'pypy' +# - 'pypy3.5-7.0' + matrix: include: - - python: "2.7" - env: - - SETUPTOOLS=setuptools PIP=pip - - python: "3.7" - env: - - SETUPTOOLS=setuptools PIP=pip - dist: xenial - sudo: true - install: - - pip install mypy - - make typecheck - - python: "3.6" - install: - - pip install mypy - - make typecheck - env: - - SETUPTOOLS=setuptools PIP=pip - - python: "3.5" - env: - - SETUPTOOLS=setuptools PIP=pip - - python: "3.4" - env: - - SETUPTOOLS=setuptools PIP=pip + - name: 'Type checking' + python: '3.7' + env: TOXENV=typecheck before_install: - - pip install $SETUPTOOLS $PIP -U + - pip install -U tox tox-travis - pip --version - pip install -r testrequirements.txt - pip freeze @@ -40,5 +31,4 @@ after_success: - coveralls # command to run tests -script: - - nosetests -v --with-coverage --cover-package=fs tests +script: tox diff --git a/CHANGELOG.md b/CHANGELOG.md index f593b91f..aa0975e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed tests leaving tmp files - Fixed typing issues - Fixed link namespace returning bytes +- Fixed abstract class import from `collections` which would break on Python 3.8 +- Fixed incorrect imports of `mock` on Python 3 +- Removed some unused imports and unused `requirements.txt` file +- Added mypy checks to Travis + +### Changed + +Entire test suite has been migrated to [pytest](https://docs.pytest.org/en/latest/). +Closes [#327](https://github.com/PyFilesystem/pyfilesystem2/issues/327). ## [2.4.10] - 2019-07-29 diff --git a/appveyor.yml b/appveyor.yml index 6cdb3773..a8f3f337 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,10 +20,10 @@ environment: install: # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install nose psutil pyftpdlib mock" + - "%PYTHON%\\python.exe -m pip install pytest pytest-randomly pytest-cov psutil pyftpdlib mock" - "%PYTHON%\\python.exe setup.py install" build: off test_script: - - "%PYTHON%\\python.exe -m nose tests -v" + - "%PYTHON%\\python.exe -m pytest -v tests" diff --git a/docs/source/implementers.rst b/docs/source/implementers.rst index 1f23866b..51c33891 100644 --- a/docs/source/implementers.rst +++ b/docs/source/implementers.rst @@ -54,6 +54,13 @@ You may also want to override some of the methods in the test suite for more tar .. autoclass:: fs.test.FSTestCases :members: +.. note:: + + As of version 2.4.11 this project uses `pytest `_ to run its tests. + While it's completely compatible with ``unittest``-style tests, it's much more powerful and + feature-rich. We suggest you take advantage of it and its plugins in new tests you write, rather + than sticking to strict ``unittest`` features. For benefits and limitations, see `here `_. + .. _essential-methods: diff --git a/fs/error_tools.py b/fs/error_tools.py index ba32c23c..f3fa7194 100644 --- a/fs/error_tools.py +++ b/fs/error_tools.py @@ -8,7 +8,6 @@ import errno import platform import sys -import typing from contextlib import contextmanager from six import reraise, PY3 @@ -116,4 +115,4 @@ def unwrap_errors(path_replace): e.path = path_replace.get(e.path, e.path) else: e.path = path_replace - reraise(type(e), e) + raise diff --git a/fs/test.py b/fs/test.py index a39ecc94..31271b6c 100644 --- a/fs/test.py +++ b/fs/test.py @@ -8,7 +8,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import collections from datetime import datetime import io import itertools @@ -17,6 +16,8 @@ import os import time +import pytest + import fs.copy import fs.move from fs import ResourceType, Seek @@ -30,6 +31,11 @@ import six from six import text_type +if six.PY2: + import collections as collections_abc +else: + import collections.abc as collections_abc + UNICODE_TEXT = """ @@ -1285,7 +1291,7 @@ def test_scandir(self): # Check scandir returns an iterable iter_scandir = self.fs.scandir("/") - self.assertTrue(isinstance(iter_scandir, collections.Iterable)) + self.assertTrue(isinstance(iter_scandir, collections_abc.Iterable)) self.assertEqual(list(iter_scandir), []) # Check scanning @@ -1298,7 +1304,7 @@ def test_scandir(self): self.fs.create("bar") self.fs.makedir("dir") iter_scandir = self.fs.scandir("/") - self.assertTrue(isinstance(iter_scandir, collections.Iterable)) + self.assertTrue(isinstance(iter_scandir, collections_abc.Iterable)) scandir = sorted( [r.raw for r in iter_scandir], key=lambda info: info["basic"]["name"] @@ -1790,7 +1796,7 @@ def test_tree(self): def test_unicode_path(self): if not self.fs.getmeta().get("unicode_paths", False): - self.skipTest("the filesystem does not support unicode paths.") + return pytest.skip("the filesystem does not support unicode paths.") self.fs.makedir("földér") self.fs.writetext("☭.txt", "Smells like communism.") @@ -1813,10 +1819,10 @@ def test_unicode_path(self): def test_case_sensitive(self): meta = self.fs.getmeta() if "case_insensitive" not in meta: - self.skipTest("case sensitivity not known") + return pytest.skip("case sensitivity not known") if meta.get("case_insensitive", False): - self.skipTest("the filesystem is not case sensitive.") + return pytest.skip("the filesystem is not case sensitive.") self.fs.makedir("foo") self.fs.makedir("Foo") @@ -1846,4 +1852,3 @@ def test_hash(self): self.assertEqual( foo_fs.hash("hashme.txt", "md5"), "9fff4bb103ab8ce4619064109c54cb9c" ) - diff --git a/fs/walk.py b/fs/walk.py index 36ef8446..11dd88c7 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -12,8 +12,6 @@ from collections import deque from collections import namedtuple -import six - from ._repr import make_repr from .errors import FSError from .path import abspath @@ -295,7 +293,7 @@ def _scan( yield info except FSError as error: if not self.on_error(dir_path, error): - six.reraise(type(error), error) + raise def walk( self, diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 0b0438f9..00000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -appdirs~=1.4.3 -backports.os==0.1.1; python_version == '2.7' -enum34==1.1.6 ; python_version < '3.4' -pytz -setuptools -six==1.10.0 -typing==3.6.4 ; python_version < '3.5' diff --git a/setup.cfg b/setup.cfg index d3766f29..230c6017 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,12 +34,18 @@ warn_return_any = false disallow_untyped_defs = false [coverage:run] +branch = true omit = fs/test.py +source = fs [coverage:report] show_missing = true +skip_covered = true exclude_lines = pragma: no cover if False: @typing.overload +[tool:pytest] +markers = + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/setup.py b/setup.py index c1bcec3f..85f87607 100644 --- a/setup.py +++ b/setup.py @@ -39,8 +39,6 @@ package_data={"fs": ["py.typed"]}, zip_safe=False, platforms=["any"], - test_suite="nose.collector", - tests_require=["appdirs", "mock", "pytz", "pyftpdlib"], url="https://github.com/PyFilesystem/pyfilesystem2", version=__version__, ) diff --git a/testrequirements.txt b/testrequirements.txt index e465d4ca..bfa0e294 100644 --- a/testrequirements.txt +++ b/testrequirements.txt @@ -1,7 +1,11 @@ -appdirs~=1.4.0 -coverage -mock -pyftpdlib==1.5.2 -python-coveralls -pytz==2016.7 -nose +pytest==4.6.5 +pytest-cov==2.7.1 +pytest-randomly==1.2.3 ; python_version<"3.5" +pytest-randomly==3.0.0 ; python_version>="3.5" +mock==3.0.5 ; python_version<"3.3" +pyftpdlib==1.5.5 + +# Not directly required. `pyftpdlib` appears to need these but doesn't list them +# as requirements. +psutil +pysendfile diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..b820712f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,34 @@ +import pytest + +try: + from unittest import mock +except ImportError: + import mock + + +@pytest.fixture +@mock.patch("appdirs.user_data_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.site_data_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.user_config_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.site_config_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.user_cache_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.user_state_dir", autospec=True, spec_set=True) +@mock.patch("appdirs.user_log_dir", autospec=True, spec_set=True) +def mock_appdir_directories( + user_log_dir_mock, + user_state_dir_mock, + user_cache_dir_mock, + site_config_dir_mock, + user_config_dir_mock, + site_data_dir_mock, + user_data_dir_mock, + tmpdir +): + """Mock out every single AppDir directory so tests can't access real ones.""" + user_log_dir_mock.return_value = str(tmpdir.join("user_log").mkdir()) + user_state_dir_mock.return_value = str(tmpdir.join("user_state").mkdir()) + user_cache_dir_mock.return_value = str(tmpdir.join("user_cache").mkdir()) + site_config_dir_mock.return_value = str(tmpdir.join("site_config").mkdir()) + user_config_dir_mock.return_value = str(tmpdir.join("user_config").mkdir()) + site_data_dir_mock.return_value = str(tmpdir.join("site_data").mkdir()) + user_data_dir_mock.return_value = str(tmpdir.join("user_data").mkdir()) diff --git a/tests/test_appfs.py b/tests/test_appfs.py index 99234258..a060e97a 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -1,26 +1,24 @@ from __future__ import unicode_literals -import unittest - +import pytest import six -from fs.appfs import UserDataFS - - -class TestAppFS(unittest.TestCase): - """Test Application FS.""" - - def test_user_data(self): - """Test UserDataFS.""" - user_data_fs = UserDataFS("fstest", "willmcgugan", "1.0") - if six.PY2: - self.assertEqual( - repr(user_data_fs), - "UserDataFS(u'fstest', author=u'willmcgugan', version=u'1.0')", - ) - else: - self.assertEqual( - repr(user_data_fs), - "UserDataFS('fstest', author='willmcgugan', version='1.0')", - ) - self.assertEqual(str(user_data_fs), "") +from fs import appfs + + +@pytest.fixture +def fs(mock_appdir_directories): + """Create a UserDataFS but strictly using a temporary directory.""" + return appfs.UserDataFS("fstest", "willmcgugan", "1.0") + + +@pytest.mark.skipif(six.PY2, reason="Test requires Python 3 repr") +def test_user_data_repr_py3(fs): + assert repr(fs) == "UserDataFS('fstest', author='willmcgugan', version='1.0')" + assert str(fs) == "" + + +@pytest.mark.skipif(not six.PY2, reason="Test requires Python 2 repr") +def test_user_data_repr_py2(fs): + assert repr(fs) == "UserDataFS(u'fstest', author=u'willmcgugan', version=u'1.0')" + assert str(fs) == "" diff --git a/tests/test_base.py b/tests/test_base.py index 66708517..188fa68d 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -4,12 +4,6 @@ import unittest -try: - import mock -except ImportError: - from unittest import mock - - from fs.base import FS from fs import errors diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 0cd91d4c..59659942 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -6,6 +6,8 @@ import tempfile import unittest +import pytest + import six import fs @@ -14,7 +16,9 @@ if platform.system() != "Windows": - @unittest.skipIf(platform.system() == "Darwin", "Bad unicode not possible on OSX") + @pytest.mark.skipif( + platform.system() == "Darwin", reason="Bad unicode not possible on OSX" + ) class TestEncoding(unittest.TestCase): TEST_FILENAME = b"foo\xb1bar" diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index 8e00a034..c649a1f4 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -1,11 +1,15 @@ from __future__ import unicode_literals -import mock import time import unittest from fs import _ftp_parse as ftp_parse +try: + from unittest import mock +except ImportError: + import mock + time2017 = time.struct_time([2017, 11, 28, 1, 1, 1, 1, 332, 0]) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 19ce22ed..139bc059 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -12,8 +12,7 @@ import unittest import uuid -from nose.plugins.attrib import attr - +import pytest from six import text_type from ftplib import error_perm @@ -128,7 +127,7 @@ def test_manager_with_host(self): self.assertEqual(str(err_info.exception), "unable to connect to ftp.example.com") -@attr("slow") +@pytest.mark.slow class TestFTPFS(FSTestCases, unittest.TestCase): user = "user" @@ -198,7 +197,6 @@ def test_geturl(self): def test_host(self): self.assertEqual(self.fs.host, self.server.host) - # @attr('slow') def test_connection_error(self): fs = FTPFS("ftp.not.a.chance", timeout=1) with self.assertRaises(errors.RemoteConnectionError): @@ -265,7 +263,7 @@ def test_features(self): pass -@attr("slow") +@pytest.mark.slow class TestAnonFTPFS(FSTestCases, unittest.TestCase): user = "anonymous" diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index aaf6e779..c8193fd6 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -3,6 +3,8 @@ import posixpath import unittest +import pytest + from fs import memoryfs from fs.test import FSTestCases from fs.test import UNICODE_TEXT @@ -28,8 +30,8 @@ def _create_many_files(self): posixpath.join(parent_dir, str(file_id)), UNICODE_TEXT ) - @unittest.skipIf( - not tracemalloc, "`tracemalloc` isn't supported on this Python version." + @pytest.mark.skipif( + not tracemalloc, reason="`tracemalloc` isn't supported on this Python version." ) def test_close_mem_free(self): """Ensure all file memory is freed when calling close(). diff --git a/tests/test_opener.py b/tests/test_opener.py index 398393f4..11bc26a5 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -1,13 +1,12 @@ from __future__ import unicode_literals import os -import mock import sys import tempfile import unittest import pkg_resources -import six +import pytest from fs import open_fs, opener from fs.osfs import OSFS @@ -17,6 +16,11 @@ from fs.opener.parse import ParseResult from fs.opener.registry import Registry +try: + from unittest import mock +except ImportError: + import mock + class TestParse(unittest.TestCase): def test_registry_repr(self): @@ -204,6 +208,7 @@ def test_manage_fs_error(self): self.assertTrue(mem_fs.isclosed()) +@pytest.mark.usefixtures("mock_appdir_directories") class TestOpeners(unittest.TestCase): def test_repr(self): # Check __repr__ works @@ -271,7 +276,7 @@ def test_open_userdata_no_version(self): self.assertEqual(app_fs.app_dirs.version, None) def test_user_data_opener(self): - user_data_fs = open_fs("userdata://fstest:willmcgugan:1.0") + user_data_fs = open_fs("userdata://fstest:willmcgugan:1.0", create=True) self.assertIsInstance(user_data_fs, UserDataFS) user_data_fs.makedir("foo", recreate=True) user_data_fs.writetext("foo/bar.txt", "baz") diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 8ed361fb..3286b87d 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -4,13 +4,13 @@ import errno import io import os -import mock import shutil import tempfile import unittest +import pytest + from fs import osfs -from fs import fsencode, fsdecode from fs.path import relpath from fs import errors @@ -18,6 +18,11 @@ from six import text_type +try: + from unittest import mock +except ImportError: + import mock + class TestOSFS(FSTestCases, unittest.TestCase): """Test OSFS implementation.""" @@ -84,7 +89,7 @@ def test_expand_vars(self): self.assertIn("TYRIONLANISTER", fs1.getsyspath("/")) self.assertNotIn("TYRIONLANISTER", fs2.getsyspath("/")) - @unittest.skipIf(osfs.sendfile is None, "sendfile not supported") + @pytest.mark.skipif(osfs.sendfile is None, reason="sendfile not supported") def test_copy_sendfile(self): # try copying using sendfile with mock.patch.object(osfs, "sendfile") as sendfile: @@ -130,7 +135,7 @@ def test_unicode_paths(self): finally: shutil.rmtree(dir_path) - @unittest.skipIf(not hasattr(os, "symlink"), "No symlink support") + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="No symlink support") def test_symlinks(self): with open(self._get_real_path("foo"), "wb") as f: f.write(b"foobar") diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index ce8794cd..dd8ad47d 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -4,16 +4,13 @@ import io import os import six -import gzip -import tarfile -import getpass import tarfile import tempfile import unittest -import uuid + +import pytest from fs import tarfs -from fs import errors from fs.enums import ResourceType from fs.compress import write_tar from fs.opener import open_fs @@ -106,7 +103,7 @@ def assert_is_bzip(self): tarfile.open(fs._tar_file, "r:{}".format(other_comps)) -@unittest.skipIf(six.PY2, "Python2 does not support LZMA") +@pytest.mark.skipif(six.PY2, reason="Python2 does not support LZMA") class TestWriteXZippedTarFS(FSTestCases, unittest.TestCase): def make_fs(self): fh, _tar_file = tempfile.mkstemp() diff --git a/tests/test_tempfs.py b/tests/test_tempfs.py index 63ce4103..f47c6129 100644 --- a/tests/test_tempfs.py +++ b/tests/test_tempfs.py @@ -4,10 +4,14 @@ from fs.tempfs import TempFS from fs import errors -import mock from .test_osfs import TestOSFS +try: + from unittest import mock +except ImportError: + import mock + class TestTempFS(TestOSFS): """Test OSFS implementation.""" diff --git a/tox.ini b/tox.ini index 2383abbb..f5a803a9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,19 +1,16 @@ [tox] -envlist = {py27,py34,py35,py36,py37}{,-scandir},pypy +envlist = {py27,py34,py35,py36,py37}{,-scandir},pypy,typecheck sitepackages = False skip_missing_interpreters=True [testenv] -deps = appdirs - backports.os - coverage - enum34 - nose - mock - pyftpdlib - pytz - psutil - scandir: scandir - pysendfile -commands = nosetests --with-coverage --cover-package=fs --cover-package=fs.opener tests \ - [] +deps = -r {toxinidir}/testrequirements.txt +commands = coverage run -m pytest --cov-append {posargs} {toxinidir}/tests + +[testenv:typecheck] +python = python37 +deps = + mypy + -r {toxinidir}/testrequirements.txt +commands = make typecheck +whitelist_externals = make From b78dad7dd382292ae6c8113f5bed96c3f660f0fd Mon Sep 17 00:00:00 2001 From: jaska Date: Fri, 23 Aug 2019 20:38:41 +0100 Subject: [PATCH 013/309] Implement TarFS.geturl and ZipFS.geturl and Fix #329, #333, #340 (#330) * :sparkles: provide geturl for ReadZipFS. As a user of zipsf, I need geturl to provide FSURL string. * :bug: on windows and python 3, fs.open_fs(osfs(~/).geturl('myfolder/subfolder')) triggers CreateFailed :bug: osfs.geturl() cannot be opened by itself * :microscope: all test cases are in and :sparkles: support geturl for read tar file system * :fire: remove unwanted comment in code * :book: update change log and contributor md * :short: update code with black * :book: update change log * :shirt: provide type info * :green_heart: update unit tests * :fire: remove dead code * :green_heart: update tarfs unit test * :fire: remove unwanted change * :short: run black over osfs.py * :bug: fix hidden exception at fs.close() when opening an absent zip/tar file URL. fix #333 * :pencil: update the behavior of geturl of zipfs and tarfs * :shirt: address review feedback :sparkles: url quote the files for proper url string * :green_heart: fix broken tests * :wheelchair: add helpful exception info to help developers, who create pypifs, gitfs, fs.datalake et al. fix #340 * :bug: fix windows path test * :sparkles: uniformly support fs purpose * :hammer: quote around the root path. #340 * :tractor: alternative file uri implementation * :microscope: try windows path test case where unicode characters stays as they are * :bug: fix unit test expectation because of the difference between windows and linux file uri * :tractor: avoid Windows File URI for fs purpose * :bug: before quote, utf8 string needs to be encoded. https://stackoverflow.com/questions/15115588/urllib-quote-throws-keyerror * :tractor: respect rfc 3986, where unicode will be quoted * :green_heart: :hammer: code refactor and fix broken unit tests * :shirt: address review feedback from @lurch * :green_heart: fix typo in code and :shirt: update assertions * :fire: remove unused variable * :shirt: address further comments from @lurch * :green_heart: update windows test case. fix the typo * :bug: colon:tmp is bad path under windows * :bug: forward slash on Windows is a valid path separator * :green_heart: fix unit tests on travis-ci * :shirt: address review comments * :shirt: mypy compliance * :shirt: dot the i and cross the t --- CHANGELOG.md | 10 +++++++-- CONTRIBUTORS.md | 1 + fs/_url_tools.py | 49 +++++++++++++++++++++++++++++++++++++++++ fs/base.py | 2 +- fs/osfs.py | 15 ++++++++----- fs/tarfs.py | 20 ++++++++++++----- fs/zipfs.py | 13 ++++++++++- tests/test_osfs.py | 45 ++++++++++++++++++++++++++++++++----- tests/test_tarfs.py | 44 +++++++++++++++++++++++++++--------- tests/test_url_tools.py | 39 ++++++++++++++++++++++++++++++++ tests/test_zipfs.py | 34 +++++++++++++++++++++++++--- 11 files changed, 239 insertions(+), 33 deletions(-) create mode 100644 fs/_url_tools.py create mode 100644 tests/test_url_tools.py diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0975e3..20d7568b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [2.4.11] - Unreleased +### Added + +- Added geturl for TarFS and ZipFS for 'fs' purpose. NoURL for 'download' purpose. +- Added helpful root path in CreateFailed exception [#340](https://github.com/PyFilesystem/pyfilesystem2/issues/340) + ### Fixed - Fixed tests leaving tmp files - Fixed typing issues - Fixed link namespace returning bytes +- Fixed broken FSURL in windows [#329](https://github.com/PyFilesystem/pyfilesystem2/issues/329) +- Fixed hidden exception at fs.close() when opening an absent zip/tar file URL [#333](https://github.com/PyFilesystem/pyfilesystem2/issues/333) - Fixed abstract class import from `collections` which would break on Python 3.8 - Fixed incorrect imports of `mock` on Python 3 - Removed some unused imports and unused `requirements.txt` file @@ -19,8 +26,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed -Entire test suite has been migrated to [pytest](https://docs.pytest.org/en/latest/). -Closes [#327](https://github.com/PyFilesystem/pyfilesystem2/issues/327). +- Entire test suite has been migrated to [pytest](https://docs.pytest.org/en/latest/). Closes [#327](https://github.com/PyFilesystem/pyfilesystem2/issues/327). ## [2.4.10] - 2019-07-29 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ef1d3a09..cb55cf74 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -2,6 +2,7 @@ Many thanks to the following developers for contributing to this project: +- [C. W.](https://github.com/chfw) - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) - [Giampaolo](https://github.com/gpcimino) diff --git a/fs/_url_tools.py b/fs/_url_tools.py new file mode 100644 index 00000000..4c6fd73f --- /dev/null +++ b/fs/_url_tools.py @@ -0,0 +1,49 @@ +import re +import six +import platform + +if False: # typing.TYPE_CHECKING + from typing import Text, Union, BinaryIO + +_WINDOWS_PLATFORM = platform.system() == "Windows" + + +def url_quote(path_snippet): + # type: (Text) -> Text + """ + On Windows, it will separate drive letter and quote windows + path alone. No magic on Unix-alie path, just pythonic + `pathname2url` + + Arguments: + path_snippet: a file path, relative or absolute. + """ + if _WINDOWS_PLATFORM and _has_drive_letter(path_snippet): + drive_letter, path = path_snippet.split(":", 1) + if six.PY2: + path = path.encode("utf-8") + path = six.moves.urllib.request.pathname2url(path) + path_snippet = "{}:{}".format(drive_letter, path) + else: + if six.PY2: + path_snippet = path_snippet.encode("utf-8") + path_snippet = six.moves.urllib.request.pathname2url(path_snippet) + return path_snippet + + +def _has_drive_letter(path_snippet): + # type: (Text) -> bool + """ + The following path will get True + D:/Data + C:\\My Dcouments\\ test + + And will get False + + /tmp/abc:test + + Arguments: + path_snippet: a file path, relative or absolute. + """ + windows_drive_pattern = ".:[/\\\\].*$" + return re.match(windows_drive_pattern, path_snippet) is not None diff --git a/fs/base.py b/fs/base.py index 18f3ccdd..fae7ce12 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1633,7 +1633,7 @@ def hash(self, path, name): fs.errors.UnsupportedHash: If the requested hash is not supported. """ - _path = self.validatepath(path) + self.validatepath(path) try: hash_object = hashlib.new(name) except ValueError: diff --git a/fs/osfs.py b/fs/osfs.py index 10f8713a..8782551a 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -39,7 +39,6 @@ sendfile = None # type: ignore # pragma: no cover from . import errors -from .errors import FileExists from .base import FS from .enums import ResourceType from ._fscompat import fsencode, fsdecode, fspath @@ -49,6 +48,7 @@ from .error_tools import convert_os_errors from .mode import Mode, validate_open_mode from .errors import FileExpected, NoURL +from ._url_tools import url_quote if False: # typing.TYPE_CHECKING from typing import ( @@ -137,7 +137,8 @@ def __init__( ) else: if not os.path.isdir(_root_path): - raise errors.CreateFailed("root path does not exist") + message = "root path '{}' does not exist".format(_root_path) + raise errors.CreateFailed(message) _meta = self._meta = { "network": False, @@ -526,7 +527,6 @@ def _scandir(self, path, namespaces=None): namespaces = namespaces or () _path = self.validatepath(path) sys_path = self.getsyspath(_path) - _sys_path = fsencode(sys_path) with convert_os_errors("scandir", path, directory=True): for entry_name in os.listdir(sys_path): _entry_name = fsdecode(entry_name) @@ -584,9 +584,14 @@ def getsyspath(self, path): def geturl(self, path, purpose="download"): # type: (Text, Text) -> Text - if purpose != "download": + sys_path = self.getsyspath(path) + if purpose == "download": + return "file://" + sys_path + elif purpose == "fs": + url_path = url_quote(sys_path) + return "osfs://" + url_path + else: raise NoURL(path, purpose) - return "file://" + self.getsyspath(path) def gettype(self, path): # type: (Text) -> ResourceType diff --git a/fs/tarfs.py b/fs/tarfs.py index ce2109c2..250291a1 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -4,7 +4,6 @@ from __future__ import print_function from __future__ import unicode_literals -import copy import os import tarfile import typing @@ -17,14 +16,14 @@ from .base import FS from .compress import write_tar from .enums import ResourceType -from .errors import IllegalBackReference +from .errors import IllegalBackReference, NoURL from .info import Info from .iotools import RawWrapper from .opener import open_fs -from .path import dirname, relpath, basename, isbase, normpath, parts, frombase +from .path import relpath, basename, isbase, normpath, parts, frombase from .wrapfs import WrapFS from .permissions import Permissions - +from ._url_tools import url_quote if False: # typing.TYPE_CHECKING from tarfile import TarInfo @@ -461,16 +460,25 @@ def removedir(self, path): def close(self): # type: () -> None super(ReadTarFS, self).close() - self._tar.close() + if hasattr(self, "_tar"): + self._tar.close() def isclosed(self): # type: () -> bool return self._tar.closed # type: ignore + def geturl(self, path, purpose="download"): + # type: (Text, Text) -> Text + if purpose == "fs" and isinstance(self._file, six.string_types): + quoted_file = url_quote(self._file) + quoted_path = url_quote(path) + return "tar://{}!/{}".format(quoted_file, quoted_path) + else: + raise NoURL(path, purpose) + if __name__ == "__main__": # pragma: no cover from fs.tree import render - from fs.opener import open_fs with TarFS("tests.tar") as tar_fs: print(tar_fs.listdir("/")) diff --git a/fs/zipfs.py b/fs/zipfs.py index 1fdf463b..c347731c 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -22,6 +22,7 @@ from .path import dirname, forcedir, normpath, relpath from .time import datetime_to_epoch from .wrapfs import WrapFS +from ._url_tools import url_quote if False: # typing.TYPE_CHECKING from typing import ( @@ -434,7 +435,8 @@ def removedir(self, path): def close(self): # type: () -> None super(ReadZipFS, self).close() - self._zip.close() + if hasattr(self, "_zip"): + self._zip.close() def readbytes(self, path): # type: (Text) -> bytes @@ -444,3 +446,12 @@ def readbytes(self, path): zip_name = self._path_to_zip_name(path) zip_bytes = self._zip.read(zip_name) return zip_bytes + + def geturl(self, path, purpose="download"): + # type: (Text, Text) -> Text + if purpose == "fs" and isinstance(self._file, six.string_types): + quoted_file = url_quote(self._file) + quoted_path = url_quote(path) + return "zip://{}!/{}".format(quoted_file, quoted_path) + else: + raise errors.NoURL(path, purpose) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 3286b87d..bd125c1e 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -7,13 +7,11 @@ import shutil import tempfile import unittest - import pytest -from fs import osfs -from fs.path import relpath +from fs import osfs, open_fs +from fs.path import relpath, dirname from fs import errors - from fs.test import FSTestCases from six import text_type @@ -77,7 +75,7 @@ def assert_text(self, path, contents): def test_not_exists(self): with self.assertRaises(errors.CreateFailed): - fs = osfs.OSFS("/does/not/exists/") + osfs.OSFS("/does/not/exists/") def test_expand_vars(self): self.fs.makedir("TYRIONLANISTER") @@ -162,3 +160,40 @@ def test_validatepath(self): with self.assertRaises(errors.InvalidCharsInPath): with self.fs.open("13 – Marked Register.pdf", "wb") as fh: fh.write(b"foo") + + def test_consume_geturl(self): + self.fs.create("foo") + try: + url = self.fs.geturl("foo", purpose="fs") + except errors.NoURL: + self.assertFalse(self.fs.hasurl("foo")) + else: + self.assertTrue(self.fs.hasurl("foo")) + + # Should not throw an error + base_dir = dirname(url) + open_fs(base_dir) + + def test_complex_geturl(self): + self.fs.makedirs("foo/bar ha") + test_fixtures = [ + # test file, expected url path + ["foo", "foo"], + ["foo-bar", "foo-bar"], + ["foo_bar", "foo_bar"], + ["foo/bar ha/barz", "foo/bar%20ha/barz"], + ["example b.txt", "example%20b.txt"], + ["exampleㄓ.txt", "example%E3%84%93.txt"], + ] + file_uri_prefix = "osfs://" + for test_file, relative_url_path in test_fixtures: + self.fs.create(test_file) + expected = file_uri_prefix + self.fs.getsyspath(relative_url_path).replace( + "\\", "/" + ) + actual = self.fs.geturl(test_file, purpose="fs") + + self.assertEqual(actual, expected) + + def test_geturl_return_no_url(self): + self.assertRaises(errors.NoURL, self.fs.geturl, "test/path", "upload") diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index dd8ad47d..c3570bdb 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -7,7 +7,6 @@ import tarfile import tempfile import unittest - import pytest from fs import tarfs @@ -15,6 +14,7 @@ from fs.compress import write_tar from fs.opener import open_fs from fs.opener.errors import NotWriteable +from fs.errors import NoURL from fs.test import FSTestCases from .test_archives import ArchiveTestCases @@ -93,15 +93,6 @@ def destroy_fs(self, fs): os.remove(fs._tar_file) del fs._tar_file - def assert_is_bzip(self): - try: - tarfile.open(fs._tar_file, "r:gz") - except tarfile.ReadError: - self.fail("{} is not a valid gz archive".format(fs._tar_file)) - for other_comps in ["xz", "bz2", ""]: - with self.assertRaises(tarfile.ReadError): - tarfile.open(fs._tar_file, "r:{}".format(other_comps)) - @pytest.mark.skipif(six.PY2, reason="Python2 does not support LZMA") class TestWriteXZippedTarFS(FSTestCases, unittest.TestCase): @@ -181,11 +172,44 @@ def test_read_from_filename(self): except: self.fail("Couldn't open tarfs from filename") + def test_read_non_existent_file(self): + fs = tarfs.TarFS(open(self._temp_path, "rb")) + # it has been very difficult to catch exception in __del__() + del fs._tar + try: + fs.close() + except AttributeError: + self.fail("Could not close tar fs properly") + except Exception: + self.fail("Strange exception in closing fs") + def test_getinfo(self): super(TestReadTarFS, self).test_getinfo() top = self.fs.getinfo("top.txt", ["tar"]) self.assertTrue(top.get("tar", "is_file")) + def test_geturl_for_fs(self): + test_fixtures = [ + # test_file, expected + ["foo/bar/egg/foofoo", "foo/bar/egg/foofoo"], + ["foo/bar egg/foo foo", "foo/bar%20egg/foo%20foo"], + ] + tar_file_path = self._temp_path.replace("\\", "/") + for test_file, expected_file in test_fixtures: + expected = "tar://{tar_file_path}!/{file_inside_tar}".format( + tar_file_path=tar_file_path, file_inside_tar=expected_file + ) + self.assertEqual(self.fs.geturl(test_file, purpose="fs"), expected) + + def test_geturl_for_fs_but_file_is_binaryio(self): + self.fs._file = six.BytesIO() + self.assertRaises(NoURL, self.fs.geturl, "test", "fs") + + def test_geturl_for_download(self): + test_file = "foo/bar/egg/foofoo" + with self.assertRaises(NoURL): + self.fs.geturl(test_file) + class TestBrokenPaths(unittest.TestCase): @classmethod diff --git a/tests/test_url_tools.py b/tests/test_url_tools.py new file mode 100644 index 00000000..5b5d4a1d --- /dev/null +++ b/tests/test_url_tools.py @@ -0,0 +1,39 @@ +# coding: utf-8 +"""Test url tools. """ +from __future__ import unicode_literals + +import platform +import unittest + +from fs._url_tools import url_quote + + +class TestBase(unittest.TestCase): + def test_quote(self): + test_fixtures = [ + # test_snippet, expected + ["foo/bar/egg/foofoo", "foo/bar/egg/foofoo"], + ["foo/bar ha/barz", "foo/bar%20ha/barz"], + ["example b.txt", "example%20b.txt"], + ["exampleㄓ.txt", "example%E3%84%93.txt"], + ] + if platform.system() == "Windows": + test_fixtures.extend( + [ + ["C:\\My Documents\\test.txt", "C:/My%20Documents/test.txt"], + ["C:/My Documents/test.txt", "C:/My%20Documents/test.txt"], + # on Windows '\' is regarded as path separator + ["test/forward\\slash", "test/forward/slash"], + ] + ) + else: + test_fixtures.extend( + [ + # colon:tmp is bad path under Windows + ["test/colon:tmp", "test/colon%3Atmp"], + # Unix treat \ as %5C + ["test/forward\\slash", "test/forward%5Cslash"], + ] + ) + for test_snippet, expected in test_fixtures: + self.assertEqual(url_quote(test_snippet), expected) diff --git a/tests/test_zipfs.py b/tests/test_zipfs.py index 421d80d8..9b2e82ea 100644 --- a/tests/test_zipfs.py +++ b/tests/test_zipfs.py @@ -13,8 +13,9 @@ from fs.compress import write_zip from fs.opener import open_fs from fs.opener.errors import NotWriteable +from fs.errors import NoURL from fs.test import FSTestCases -from fs.enums import Seek, ResourceType +from fs.enums import Seek from .test_archives import ArchiveTestCases @@ -168,6 +169,33 @@ def test_seek_end(self): self.assertEqual(f.seek(-5, Seek.end), 7) self.assertEqual(f.read(), b"World") + def test_geturl_for_fs(self): + test_file = "foo/bar/egg/foofoo" + expected = "zip://{zip_file_path}!/{file_inside_zip}".format( + zip_file_path=self._temp_path.replace("\\", "/"), file_inside_zip=test_file + ) + self.assertEqual(self.fs.geturl(test_file, purpose="fs"), expected) + + def test_geturl_for_fs_but_file_is_binaryio(self): + self.fs._file = six.BytesIO() + self.assertRaises(NoURL, self.fs.geturl, "test", "fs") + + def test_geturl_for_download(self): + test_file = "foo/bar/egg/foofoo" + with self.assertRaises(NoURL): + self.fs.geturl(test_file) + + def test_read_non_existent_file(self): + fs = zipfs.ZipFS(open(self._temp_path, "rb")) + # it has been very difficult to catch exception in __del__() + del fs._zip + try: + fs.close() + except AttributeError: + self.fail("Could not close tar fs properly") + except Exception: + self.fail("Strange exception in closing fs") + class TestReadZipFSMem(TestReadZipFS): def make_source_fs(self): @@ -184,8 +212,8 @@ def test_implied(self): z.writestr("foo/bar/baz/egg", b"hello") with zipfs.ReadZipFS(path) as zip_fs: foo = zip_fs.getinfo("foo", ["details"]) - bar = zip_fs.getinfo("foo/bar") - baz = zip_fs.getinfo("foo/bar/baz") + self.assertEqual(zip_fs.getinfo("foo/bar").name, "bar") + self.assertEqual(zip_fs.getinfo("foo/bar/baz").name, "baz") self.assertTrue(foo.is_dir) self.assertTrue(zip_fs.isfile("foo/bar/baz/egg")) finally: From 2d9189d40847bb91107173292ea36ca5f486e864 Mon Sep 17 00:00:00 2001 From: Diego Argueta Date: Fri, 30 Aug 2019 01:09:19 -0700 Subject: [PATCH 014/309] Fix missing ENOTSUP on PyPy (#339) * Fix missing ENOTSUP on PyPy * Update CHANGELOG * Start testing PyPy * Attempt to fix Travis failure * Remove unnecessary typing * Fix PyPy3.5 RLock crash --- CHANGELOG.md | 1 + fs/ftpfs.py | 5 ++--- fs/osfs.py | 23 ++++++++++++----------- setup.py | 2 ++ tests/test_osfs.py | 2 +- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d7568b..f5e2c72d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed incorrect imports of `mock` on Python 3 - Removed some unused imports and unused `requirements.txt` file - Added mypy checks to Travis +- Fixed missing `errno.ENOTSUP` on PyPy. Closes [#338](https://github.com/PyFilesystem/pyfilesystem2/issues/338). ### Changed diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 21e98a10..8ee1cb79 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -728,9 +728,8 @@ def _scandir(self, path, namespaces=None): for raw_info in self._parse_mlsx(lines): yield Info(raw_info) return - with self._lock: - for info in self._read_dir(_path).values(): - yield info + for info in self._read_dir(_path).values(): + yield info def scandir( self, diff --git a/fs/osfs.py b/fs/osfs.py index 8782551a..ce6159ad 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -421,17 +421,18 @@ def _check_copy(self, src_path, dst_path, overwrite=False): if sys.version_info[:2] < (3, 8) and sendfile is not None: - _sendfile_error_codes = frozenset( - { - errno.EIO, - errno.EINVAL, - errno.ENOSYS, - errno.ENOTSUP, # type: ignore - errno.EBADF, - errno.ENOTSOCK, - errno.EOPNOTSUPP, - } - ) + _sendfile_error_codes = { + errno.EIO, + errno.EINVAL, + errno.ENOSYS, + errno.EBADF, + errno.ENOTSOCK, + errno.EOPNOTSUPP, + } + + # PyPy doesn't define ENOTSUP so we have to add it conditionally. + if hasattr(errno, "ENOTSUP"): + _sendfile_error_codes.add(errno.ENOTSUP) def copy(self, src_path, dst_path, overwrite=False): # type: (Text, Text, bool) -> None diff --git a/setup.py b/setup.py index 85f87607..d758fca4 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,8 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", "Topic :: System :: Filesystems", ] diff --git a/tests/test_osfs.py b/tests/test_osfs.py index bd125c1e..18eabd58 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -91,7 +91,7 @@ def test_expand_vars(self): def test_copy_sendfile(self): # try copying using sendfile with mock.patch.object(osfs, "sendfile") as sendfile: - sendfile.side_effect = OSError(errno.ENOTSUP, "sendfile not supported") + sendfile.side_effect = OSError(errno.ENOSYS, "sendfile not supported") self.test_copy() # check other errors are transmitted self.fs.touch("foo") From 6f89b81daab7b604a77eff232639b03b6fc55ab2 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 31 Aug 2019 18:48:05 +0100 Subject: [PATCH 015/309] add 3.8dev (#352) * add 3.8dev * remove landscape * allow 3.8 * requirements for readthedocs.io * badges on two lines * added 3.8 * changelog * requirements for readthedocs --- .travis.yml | 15 ++++++++------- CHANGELOG.md | 1 + README.md | 4 +++- fs/osfs.py | 2 +- requirements-readthedocs.txt | 2 ++ setup.py | 1 + 6 files changed, 16 insertions(+), 9 deletions(-) create mode 100644 requirements-readthedocs.txt diff --git a/.travis.yml b/.travis.yml index d435957b..1e19a196 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,19 +3,20 @@ sudo: false language: python python: - - '2.7' - - '3.4' - - '3.5' - - '3.6' - - '3.7' + - "2.7" + - "3.4" + - "3.5" + - "3.6" + - "3.7" + - "3.8-dev" # TODO (dargueta): Enable these once we figure out FTP timing issues in PyPy tests # - 'pypy' # - 'pypy3.5-7.0' matrix: include: - - name: 'Type checking' - python: '3.7' + - name: "Type checking" + python: "3.7" env: TOXENV=typecheck before_install: diff --git a/CHANGELOG.md b/CHANGELOG.md index f5e2c72d..6967df4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added geturl for TarFS and ZipFS for 'fs' purpose. NoURL for 'download' purpose. - Added helpful root path in CreateFailed exception [#340](https://github.com/PyFilesystem/pyfilesystem2/issues/340) +- Added Python 3.8 support ### Fixed diff --git a/README.md b/README.md index 8616bc3b..fbd6b200 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,13 @@ Python's Filesystem abstraction layer. [![PyPI version](https://badge.fury.io/py/fs.svg)](https://badge.fury.io/py/fs) [![PyPI](https://img.shields.io/pypi/pyversions/fs.svg)](https://pypi.org/project/fs/) +[![Downloads](https://pepy.tech/badge/fs/month)](https://pepy.tech/project/fs/month) + + [![Build Status](https://travis-ci.org/PyFilesystem/pyfilesystem2.svg?branch=master)](https://travis-ci.org/PyFilesystem/pyfilesystem2) [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/pyfilesystem/pyfilesystem2?branch=master&svg=true)](https://ci.appveyor.com/project/willmcgugan/pyfilesystem2) [![Coverage Status](https://coveralls.io/repos/github/PyFilesystem/pyfilesystem2/badge.svg)](https://coveralls.io/github/PyFilesystem/pyfilesystem2) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/30ad6445427349218425d93886ade9ee)](https://www.codacy.com/app/will-mcgugan/pyfilesystem2?utm_source=github.com&utm_medium=referral&utm_content=PyFilesystem/pyfilesystem2&utm_campaign=Badge_Grade) -[![Code Health](https://landscape.io/github/PyFilesystem/pyfilesystem2/master/landscape.svg?style=flat)](https://landscape.io/github/PyFilesystem/pyfilesystem2/master) ## Documentation diff --git a/fs/osfs.py b/fs/osfs.py index ce6159ad..2a711bd7 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -419,7 +419,7 @@ def _check_copy(self, src_path, dst_path, overwrite=False): raise errors.DirectoryExpected(dirname(dst_path)) return _src_path, _dst_path - if sys.version_info[:2] < (3, 8) and sendfile is not None: + if sys.version_info[:2] <= (3, 8) and sendfile is not None: _sendfile_error_codes = { errno.EIO, diff --git a/requirements-readthedocs.txt b/requirements-readthedocs.txt new file mode 100644 index 00000000..3e63add2 --- /dev/null +++ b/requirements-readthedocs.txt @@ -0,0 +1,2 @@ +# requirements for readthedocs.io +-e . diff --git a/setup.py b/setup.py index d758fca4..a6ce6f43 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,7 @@ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: System :: Filesystems", From 29a9e64e0ea7b3b0246c930a58da232fd8ead4a1 Mon Sep 17 00:00:00 2001 From: Diego Argueta Date: Sat, 7 Sep 2019 07:53:57 -0700 Subject: [PATCH 016/309] Add flake8 linting (#351) * Remove unused imports, fix line lengths * Fix JSON exception catching * Add linting tools * More linting * Fix broken/extra imports, long lines, unused variables * Fix hasattr/getattr confusion * Remove unused loop variables, fix comprehensions * Update CHANGELOG * Move tool configs to setup.cfg with the rest --- .travis.yml | 3 +++ CHANGELOG.md | 7 +++++- fs/_bulk.py | 7 +++--- fs/_fscompat.py | 2 -- fs/_repr.py | 2 +- fs/_typing.py | 2 +- fs/_url_tools.py | 5 +++-- fs/appfs.py | 2 +- fs/base.py | 7 +++--- fs/compress.py | 4 ++-- fs/copy.py | 3 +-- fs/error_tools.py | 14 ++++++------ fs/errors.py | 2 +- fs/filesize.py | 5 +++-- fs/ftpfs.py | 19 ++++++++++------ fs/glob.py | 7 +++--- fs/info.py | 18 +++++++-------- fs/iotools.py | 5 ++--- fs/memoryfs.py | 18 +++++++-------- fs/mirror.py | 3 +-- fs/mode.py | 4 ++-- fs/mountfs.py | 2 +- fs/move.py | 2 +- fs/multifs.py | 2 +- fs/opener/appfs.py | 2 +- fs/opener/base.py | 4 ++-- fs/opener/ftpfs.py | 10 ++++----- fs/opener/memoryfs.py | 4 ++-- fs/opener/osfs.py | 4 ++-- fs/opener/parse.py | 2 +- fs/opener/registry.py | 6 +---- fs/opener/tarfs.py | 4 ++-- fs/opener/tempfs.py | 4 ++-- fs/opener/zipfs.py | 4 ++-- fs/osfs.py | 13 +++++------ fs/path.py | 4 ++-- fs/permissions.py | 4 ++-- fs/subfs.py | 4 ++-- fs/tarfs.py | 12 +++++----- fs/tempfs.py | 2 +- fs/test.py | 10 ++++----- fs/tools.py | 3 +-- fs/tree.py | 2 +- fs/walk.py | 2 +- fs/wildcard.py | 5 ++--- fs/wrap.py | 6 ++--- fs/wrapfs.py | 5 +---- fs/zipfs.py | 4 ++-- setup.cfg | 17 ++++++++++++++ tests/test_archives.py | 1 - tests/test_copy.py | 4 +--- tests/test_errors.py | 2 +- tests/test_ftpfs.py | 51 ++++++++++++++++++++++++++++++------------ tests/test_opener.py | 2 +- tests/test_path.py | 26 +++++++++++++++++++-- tests/test_tarfs.py | 4 ++-- tests/test_tree.py | 1 + tox.ini | 13 ++++++++++- 58 files changed, 227 insertions(+), 159 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1e19a196..8d584b1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,9 @@ matrix: - name: "Type checking" python: "3.7" env: TOXENV=typecheck + - name: 'Lint' + python: '3.7' + env: TOXENV=lint before_install: - pip install -U tox tox-travis diff --git a/CHANGELOG.md b/CHANGELOG.md index 6967df4f..f423301e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,12 +23,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed abstract class import from `collections` which would break on Python 3.8 - Fixed incorrect imports of `mock` on Python 3 - Removed some unused imports and unused `requirements.txt` file -- Added mypy checks to Travis +- Added mypy checks to Travis. Closes [#332](https://github.com/PyFilesystem/pyfilesystem2/issues/332). - Fixed missing `errno.ENOTSUP` on PyPy. Closes [#338](https://github.com/PyFilesystem/pyfilesystem2/issues/338). +- Fixed bug in a decorator that would trigger an `AttributeError` when a class + was created that implemented a deprecated method and had no docstring of its + own. ### Changed - Entire test suite has been migrated to [pytest](https://docs.pytest.org/en/latest/). Closes [#327](https://github.com/PyFilesystem/pyfilesystem2/issues/327). +- Style checking is now enforced using `flake8`; this involved some code cleanup + such as removing unused imports. ## [2.4.10] - 2019-07-29 diff --git a/fs/_bulk.py b/fs/_bulk.py index a11069e8..9b0b8b79 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import threading +import typing from six.moves.queue import Queue @@ -14,10 +15,10 @@ from .errors import BulkCopyFailed from .tools import copy_file_data -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from .base import FS from types import TracebackType - from typing import IO, Iterator, List, Optional, Mapping, Text, Type, Union + from typing import IO, List, Optional, Text, Type class _Worker(threading.Thread): @@ -96,7 +97,7 @@ def start(self): def stop(self): """Stop the workers (will block until they are finished).""" if self.running and self.num_workers: - for worker in self.workers: + for _worker in self.workers: self.queue.put(None) for worker in self.workers: worker.join() diff --git a/fs/_fscompat.py b/fs/_fscompat.py index ca7f5431..de59fa29 100644 --- a/fs/_fscompat.py +++ b/fs/_fscompat.py @@ -1,5 +1,3 @@ -import sys - import six try: diff --git a/fs/_repr.py b/fs/_repr.py index af51c28a..0a207207 100644 --- a/fs/_repr.py +++ b/fs/_repr.py @@ -5,7 +5,7 @@ import typing -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text, Tuple diff --git a/fs/_typing.py b/fs/_typing.py index 47d6c9b1..7c1f2275 100644 --- a/fs/_typing.py +++ b/fs/_typing.py @@ -12,7 +12,7 @@ if _PY.major == 3 and _PY.minor == 5 and _PY.micro in (0, 1): - def overload(func): # pragma: no cover + def overload(func): # pragma: no cover # noqa: F811 return func diff --git a/fs/_url_tools.py b/fs/_url_tools.py index 4c6fd73f..64c58bd6 100644 --- a/fs/_url_tools.py +++ b/fs/_url_tools.py @@ -1,9 +1,10 @@ import re import six import platform +import typing -if False: # typing.TYPE_CHECKING - from typing import Text, Union, BinaryIO +if typing.TYPE_CHECKING: + from typing import Text _WINDOWS_PLATFORM = platform.system() == "Windows" diff --git a/fs/appfs.py b/fs/appfs.py index 0657faf5..dafe2e98 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -15,7 +15,7 @@ from ._repr import make_repr from appdirs import AppDirs -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Optional, Text diff --git a/fs/base.py b/fs/base.py index fae7ce12..a4b7aefb 100644 --- a/fs/base.py +++ b/fs/base.py @@ -28,7 +28,7 @@ from .time import datetime_to_epoch from .walk import Walker -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from datetime import datetime from threading import RLock from typing import ( @@ -84,7 +84,7 @@ def _method(*args, **kwargs): """.format( method.__name__ ) - if getattr(_method, "__doc__"): + if hasattr(_method, "__doc__"): _method.__doc__ += deprecated_msg return _method @@ -1624,7 +1624,8 @@ def hash(self, path, name): Arguments: path(str): A path on the filesystem. - name(str): One of the algorithms supported by the hashlib module, e.g. `"md5"` + name(str): + One of the algorithms supported by the hashlib module, e.g. `"md5"` Returns: str: The hex digest of the hash. diff --git a/fs/compress.py b/fs/compress.py index cf0f130a..2110403b 100644 --- a/fs/compress.py +++ b/fs/compress.py @@ -22,8 +22,8 @@ from .errors import NoSysPath, MissingInfoNamespace from .walk import Walker -if False: # typing.TYPE_CHECKING - from typing import BinaryIO, Optional, Text, Tuple, Type, Union +if typing.TYPE_CHECKING: + from typing import BinaryIO, Optional, Text, Tuple, Union from .base import FS ZipTime = Tuple[int, int, int, int, int, int] diff --git a/fs/copy.py b/fs/copy.py index 9b171d32..80fcdc6b 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -11,10 +11,9 @@ from .tools import is_thread_safe from .walk import Walker -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Callable, Optional, Text, Union from .base import FS - from .walk import Walker _OnCopy = Callable[[FS, Text, FS, Text], object] diff --git a/fs/error_tools.py b/fs/error_tools.py index f3fa7194..28c200bf 100644 --- a/fs/error_tools.py +++ b/fs/error_tools.py @@ -4,24 +4,24 @@ from __future__ import print_function from __future__ import unicode_literals -import collections import errno import platform import sys +import typing from contextlib import contextmanager -from six import reraise, PY3 +from six import reraise from . import errors -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from types import TracebackType - from typing import Iterator, Optional, Mapping, Text, Type, Union + from typing import Iterator, Optional, Text, Type, Union -if PY3: +try: from collections.abc import Mapping -else: - from collections import Mapping +except ImportError: + from collections import Mapping # noqa: E811 _WINDOWS_PLATFORM = platform.system() == "Windows" diff --git a/fs/errors.py b/fs/errors.py index e5452e06..b70b62e3 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -17,7 +17,7 @@ import six from six import text_type -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Optional, Text diff --git a/fs/filesize.py b/fs/filesize.py index ff2ecf63..a80fd9e1 100644 --- a/fs/filesize.py +++ b/fs/filesize.py @@ -16,7 +16,7 @@ import typing -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Iterable, SupportsInt, Text @@ -34,7 +34,8 @@ def _to_str(size, suffixes, base): elif size < base: return "{:,} bytes".format(size) - for i, suffix in enumerate(suffixes, 2): + # TODO (dargueta): Don't rely on unit or suffix being defined in the loop. + for i, suffix in enumerate(suffixes, 2): # noqa: B007 unit = base ** i if size < unit: break diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 8ee1cb79..11d2c4cc 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals import calendar -import ftplib import io import itertools import socket @@ -36,7 +35,7 @@ from .path import split from . import _ftp_parse as ftp_parse -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: import ftplib from typing import ( Any, @@ -45,7 +44,6 @@ ContextManager, Iterable, Iterator, - Collection, Container, Dict, List, @@ -103,7 +101,7 @@ def manage_ftp(ftp): finally: try: ftp.quit() - except: # pragma: no cover + except Exception: # pragma: no cover pass @@ -442,8 +440,15 @@ def _manage_ftp(self): def ftp_url(self): # type: () -> Text """Get the FTP url this filesystem will open.""" - _host_part = self.host if self.port == 21 else "{}:{}".format(self.host, self.port) - _user_part = "" if self.user == "anonymous" or self.user is None else "{}:{}@".format(self.user, self.passwd) + if self.port == 21: + _host_part = self.host + else: + _host_part = "{}:{}".format(self.host, self.port) + + if self.user == "anonymous" or self.user is None: + _user_part = "" + else: + _user_part = "{}:{}@".format(self.user, self.passwd) url = "ftp://{}{}".format(_user_part, _host_part) return url @@ -575,7 +580,7 @@ def _parse_mlsx(cls, lines): details["created"] = cls._parse_ftp_time(facts["create"]) yield raw_info - if False: # typing.TYPE_CHECKING + if typing.TYPE_CHECKING: def opendir(self, path, factory=None): # type: (_F, Text, Optional[_OpendirFactory]) -> SubFS[_F] diff --git a/fs/glob.py b/fs/glob.py index bff4790f..ac12125e 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals from collections import namedtuple -from typing import Iterator, List import re +import typing from .lrucache import LRUCache from ._repr import make_repr @@ -14,10 +14,9 @@ Counts = namedtuple("Counts", ["files", "directories", "data"]) LineCounts = namedtuple("LineCounts", ["lines", "non_blank"]) -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Iterator, List, Optional, Pattern, Text, Tuple from .base import FS - from .info import Info _PATTERN_CACHE = LRUCache( @@ -180,7 +179,7 @@ def count(self): directories = 0 files = 0 data = 0 - for path, info in self._make_iter(namespaces=["details"]): + for _path, info in self._make_iter(namespaces=["details"]): if info.is_dir: directories += 1 else: diff --git a/fs/info.py b/fs/info.py index 194dd48f..13f7b17f 100644 --- a/fs/info.py +++ b/fs/info.py @@ -18,7 +18,7 @@ from .time import epoch_to_datetime from ._typing import overload, Text -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from datetime import datetime from typing import Any, Callable, List, Mapping, Optional, Union @@ -69,16 +69,16 @@ def __eq__(self, other): return self.raw == getattr(other, "raw", None) @overload - def _make_datetime(self, t): # pragma: no cover + def _make_datetime(self, t): # type: (None) -> None pass - @overload - def _make_datetime(self, t): # pragma: no cover + @overload # noqa: F811 + def _make_datetime(self, t): # type: (int) -> datetime pass - def _make_datetime(self, t): + def _make_datetime(self, t): # noqa: F811 # type: (Optional[int]) -> Optional[datetime] if t is not None: return self._to_datetime(t) @@ -86,16 +86,16 @@ def _make_datetime(self, t): return None @overload - def get(self, namespace, key): # pragma: no cover + def get(self, namespace, key): # type: (Text, Text) -> Any pass - @overload - def get(self, namespace, key, default): # pragma: no cover + @overload # noqa: F811 + def get(self, namespace, key, default): # type: (Text, Text, T) -> Union[Any, T] pass - def get(self, namespace, key, default=None): + def get(self, namespace, key, default=None): # noqa: F811 # type: (Text, Text, Optional[Any]) -> Optional[Any] """Get a raw info value. diff --git a/fs/iotools.py b/fs/iotools.py index 87e6997a..26402ff3 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -10,11 +10,10 @@ from .mode import Mode -if False: # typing.TYPE_CHECKING - from io import RawIOBase, IOBase +if typing.TYPE_CHECKING: + from io import RawIOBase from typing import ( Any, - BinaryIO, Iterable, Iterator, IO, diff --git a/fs/memoryfs.py b/fs/memoryfs.py index dcc03050..0814b2e0 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -23,7 +23,7 @@ from .path import split from ._typing import overload -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, BinaryIO, @@ -228,22 +228,22 @@ def size(self): _bytes_file.seek(0, os.SEEK_END) return _bytes_file.tell() - @overload - def get_entry(self, name, default): # pragma: no cover + @overload # noqa: F811 + def get_entry(self, name, default): # type: (Text, _DirEntry) -> _DirEntry pass - @overload - def get_entry(self, name): # pragma: no cover + @overload # noqa: F811 + def get_entry(self, name): # type: (Text) -> Optional[_DirEntry] pass - @overload - def get_entry(self, name, default): # pragma: no cover + @overload # noqa: F811 + def get_entry(self, name, default): # type: (Text, None) -> Optional[_DirEntry] pass - def get_entry(self, name, default=None): + def get_entry(self, name, default=None): # noqa: F811 # type: (Text, Optional[_DirEntry]) -> Optional[_DirEntry] assert self.is_dir, "must be a directory" return self._dir.get(name, default) @@ -377,7 +377,7 @@ def listdir(self, path): raise errors.DirectoryExpected(path) return dir_entry.list() - if False: # typing.TYPE_CHECKING + if typing.TYPE_CHECKING: def opendir(self, path, factory=None): # type: (_M, Text, Optional[_OpendirFactory]) -> SubFS[_M] diff --git a/fs/mirror.py b/fs/mirror.py index 98f3d5f1..ceb8ccd3 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -19,7 +19,6 @@ from __future__ import print_function from __future__ import unicode_literals -from contextlib import contextmanager import typing from ._bulk import Copier @@ -29,7 +28,7 @@ from .tools import is_thread_safe from .walk import Walker -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Callable, Optional, Text, Union from .base import FS from .info import Info diff --git a/fs/mode.py b/fs/mode.py index 76b665d6..5e8c795d 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -15,8 +15,8 @@ from ._typing import Text -if False: # typing.TYPE_CHECKING - from typing import Container, FrozenSet, Set, Union +if typing.TYPE_CHECKING: + from typing import FrozenSet, Set, Union __all__ = ["Mode", "check_readable", "check_writable", "validate_openbin_mode"] diff --git a/fs/mountfs.py b/fs/mountfs.py index aa314ed5..d51d7d9d 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -18,7 +18,7 @@ from .mode import validate_open_mode from .mode import validate_openbin_mode -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, BinaryIO, diff --git a/fs/move.py b/fs/move.py index 5da6b82d..4f6fc2ab 100644 --- a/fs/move.py +++ b/fs/move.py @@ -10,7 +10,7 @@ from .copy import copy_file from .opener import manage_fs -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from .base import FS from typing import Text, Union diff --git a/fs/multifs.py b/fs/multifs.py index 60a3ad40..e68d2c00 100644 --- a/fs/multifs.py +++ b/fs/multifs.py @@ -17,7 +17,7 @@ from .opener import open_fs from .path import abspath, normpath -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, BinaryIO, diff --git a/fs/opener/appfs.py b/fs/opener/appfs.py index 93cffef0..fccf603e 100644 --- a/fs/opener/appfs.py +++ b/fs/opener/appfs.py @@ -12,7 +12,7 @@ from .registry import registry from .errors import OpenerError -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text, Union from .parse import ParseResult from ..appfs import _AppFS diff --git a/fs/opener/base.py b/fs/opener/base.py index 952eea53..cd970399 100644 --- a/fs/opener/base.py +++ b/fs/opener/base.py @@ -7,8 +7,8 @@ import six -if False: # typing.TYPE_CHECKING - from typing import List, Text, Union +if typing.TYPE_CHECKING: + from typing import List, Text from ..base import FS from .parse import ParseResult diff --git a/fs/opener/ftpfs.py b/fs/opener/ftpfs.py index 4c30c962..f5beab21 100644 --- a/fs/opener/ftpfs.py +++ b/fs/opener/ftpfs.py @@ -6,17 +6,15 @@ from __future__ import print_function from __future__ import unicode_literals -import six - import typing from .base import Opener from .registry import registry -from ..errors import FSError, CreateFailed +from ..errors import CreateFailed -if False: # typing.TYPE_CHECKING - from typing import List, Text, Union - from ..ftpfs import FTPFS +if typing.TYPE_CHECKING: + from typing import Text, Union + from ..ftpfs import FTPFS # noqa: F401 from ..subfs import SubFS from .parse import ParseResult diff --git a/fs/opener/memoryfs.py b/fs/opener/memoryfs.py index 8b8976c3..696ee06a 100644 --- a/fs/opener/memoryfs.py +++ b/fs/opener/memoryfs.py @@ -11,10 +11,10 @@ from .base import Opener from .registry import registry -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text from .parse import ParseResult - from ..memoryfs import MemoryFS + from ..memoryfs import MemoryFS # noqa: F401 @registry.install diff --git a/fs/opener/osfs.py b/fs/opener/osfs.py index 986de249..00cb63ee 100644 --- a/fs/opener/osfs.py +++ b/fs/opener/osfs.py @@ -11,10 +11,10 @@ from .base import Opener from .registry import registry -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text from .parse import ParseResult - from ..osfs import OSFS + from ..osfs import OSFS # noqa: F401 @registry.install diff --git a/fs/opener/parse.py b/fs/opener/parse.py index 61f99f2c..e9423807 100644 --- a/fs/opener/parse.py +++ b/fs/opener/parse.py @@ -14,7 +14,7 @@ from .errors import ParseError -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Optional, Text diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 0ed74339..50f2976c 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -10,20 +10,18 @@ import contextlib import typing -import six import pkg_resources from .base import Opener from .errors import UnsupportedProtocol, EntryPointError from .parse import parse_fs_url -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Callable, Dict, Iterator, List, - Optional, Text, Type, Tuple, @@ -279,8 +277,6 @@ def manage_fs( _fs = self.open_fs(fs_url, create=create, writeable=writeable, cwd=cwd) try: yield _fs - except: - raise finally: _fs.close() diff --git a/fs/opener/tarfs.py b/fs/opener/tarfs.py index 867ca80f..3ff91f55 100644 --- a/fs/opener/tarfs.py +++ b/fs/opener/tarfs.py @@ -12,10 +12,10 @@ from .registry import registry from .errors import NotWriteable -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text from .parse import ParseResult - from ..tarfs import TarFS + from ..tarfs import TarFS # noqa: F401 @registry.install diff --git a/fs/opener/tempfs.py b/fs/opener/tempfs.py index 5fe47a08..ffa17983 100644 --- a/fs/opener/tempfs.py +++ b/fs/opener/tempfs.py @@ -11,10 +11,10 @@ from .base import Opener from .registry import registry -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text from .parse import ParseResult - from ..tempfs import TempFS + from ..tempfs import TempFS # noqa: F401 @registry.install diff --git a/fs/opener/zipfs.py b/fs/opener/zipfs.py index 714fe384..81e48455 100644 --- a/fs/opener/zipfs.py +++ b/fs/opener/zipfs.py @@ -12,10 +12,10 @@ from .registry import registry from .errors import NotWriteable -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Text from .parse import ParseResult - from ..zipfs import ZipFS + from ..zipfs import ZipFS # noqa: F401 @registry.install diff --git a/fs/osfs.py b/fs/osfs.py index 2a711bd7..ec68d0ad 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -50,11 +50,10 @@ from .errors import FileExpected, NoURL from ._url_tools import url_quote -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, BinaryIO, - Callable, Collection, Dict, Iterator, @@ -151,7 +150,8 @@ def __init__( try: # https://stackoverflow.com/questions/7870041/check-if-file-system-is-case-insensitive-in-python - # I don't know of a better way of detecting case insensitivity of a filesystem + # I don't know of a better way of detecting case insensitivity of a + # filesystem with tempfile.NamedTemporaryFile(prefix="TmP") as _tmp_file: _meta["case_insensitive"] = os.path.exists(_tmp_file.name.lower()) except Exception: @@ -396,7 +396,7 @@ def removedir(self, path): # --- Type hint for opendir ------------------------------ - if False: # typing.TYPE_CHECKING + if typing.TYPE_CHECKING: def opendir(self, path, factory=None): # type: (_O, Text, Optional[_OpendirFactory]) -> SubFS[_O] @@ -672,8 +672,7 @@ def validatepath(self, path): except UnicodeEncodeError as error: raise errors.InvalidCharsInPath( path, - msg="path '{path}' could not be encoded for the filesystem (check LANG env var); {error}".format( - path=path, error=error - ), + msg="path '{path}' could not be encoded for the filesystem (check LANG" + " env var); {error}".format(path=path, error=error), ) return super(OSFS, self).validatepath(path) diff --git a/fs/path.py b/fs/path.py index 3783dd13..49f04bd6 100644 --- a/fs/path.py +++ b/fs/path.py @@ -16,7 +16,7 @@ from .errors import IllegalBackReference -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import List, Text, Tuple @@ -68,7 +68,7 @@ def normpath(path): ... IllegalBackReference: path 'foo/../../bar' contains back-references outside of filesystem" - """ + """ # noqa: E501 if path in "/": return path diff --git a/fs/permissions.py b/fs/permissions.py index 7c8b2030..19934465 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -5,14 +5,14 @@ from __future__ import unicode_literals import typing -from typing import Container, Iterable +from typing import Iterable import six from ._typing import Text -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Iterator, List, Optional, Tuple, Type, Union diff --git a/fs/subfs.py b/fs/subfs.py index d0d0d386..7172008e 100644 --- a/fs/subfs.py +++ b/fs/subfs.py @@ -11,9 +11,9 @@ from .wrapfs import WrapFS from .path import abspath, join, normpath, relpath -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: + from .base import FS # noqa: F401 from typing import Text, Tuple - from .base import FS _F = typing.TypeVar("_F", bound="FS", covariant=True) diff --git a/fs/tarfs.py b/fs/tarfs.py index 250291a1..20073c9d 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -20,16 +20,15 @@ from .info import Info from .iotools import RawWrapper from .opener import open_fs -from .path import relpath, basename, isbase, normpath, parts, frombase -from .wrapfs import WrapFS from .permissions import Permissions from ._url_tools import url_quote +from .path import relpath, basename, isbase, normpath, parts, frombase +from .wrapfs import WrapFS -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from tarfile import TarInfo from typing import ( Any, - AnyStr, BinaryIO, Collection, Dict, @@ -39,8 +38,7 @@ Tuple, Union, ) - from .info import Info, RawInfo - from .permissions import Permissions + from .info import RawInfo from .subfs import SubFS T = typing.TypeVar("T", bound="ReadTarFS") @@ -143,7 +141,7 @@ def __new__( else: return ReadTarFS(file, encoding=encoding) - if False: # typing.TYPE_CHECKING + if typing.TYPE_CHECKING: def __init__( self, diff --git a/fs/tempfs.py b/fs/tempfs.py index 293a692e..748463c1 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -21,7 +21,7 @@ from . import errors from .osfs import OSFS -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import Optional, Text diff --git a/fs/test.py b/fs/test.py index 31271b6c..146938a8 100644 --- a/fs/test.py +++ b/fs/test.py @@ -485,8 +485,8 @@ def test_getinfo(self): # Raw info should be serializable try: json.dumps(info) - except: - assert False, "info should be JSON serializable" + except (TypeError, ValueError): + raise AssertionError("info should be JSON serializable") # Non existant namespace is not an error no_info = self.fs.getinfo("foo", "__nosuchnamespace__").raw @@ -1286,7 +1286,7 @@ def test_desc(self): def test_scandir(self): # Check exception for scanning dir that doesn't exist with self.assertRaises(errors.ResourceNotFound): - for info in self.fs.scandir("/foobar"): + for _info in self.fs.scandir("/foobar"): pass # Check scandir returns an iterable @@ -1307,7 +1307,7 @@ def test_scandir(self): self.assertTrue(isinstance(iter_scandir, collections_abc.Iterable)) scandir = sorted( - [r.raw for r in iter_scandir], key=lambda info: info["basic"]["name"] + (r.raw for r in iter_scandir), key=lambda info: info["basic"]["name"] ) # Filesystems may send us more than we ask for @@ -1337,7 +1337,7 @@ def test_scandir(self): self.assertEqual(len(page2), 1) page3 = list(self.fs.scandir("/", page=(4, 6))) self.assertEqual(len(page3), 0) - paged = set(r.name for r in itertools.chain(page1, page2)) + paged = {r.name for r in itertools.chain(page1, page2)} self.assertEqual(paged, {"foo", "bar", "dir"}) def test_filterdir(self): diff --git a/fs/tools.py b/fs/tools.py index 4b842029..8a16d289 100644 --- a/fs/tools.py +++ b/fs/tools.py @@ -4,7 +4,6 @@ from __future__ import print_function from __future__ import unicode_literals -import io import typing from . import errors @@ -15,7 +14,7 @@ from .path import normpath from .path import recursepath -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import IO, List, Optional, Text from .base import FS diff --git a/fs/tree.py b/fs/tree.py index a10beca9..faee5472 100644 --- a/fs/tree.py +++ b/fs/tree.py @@ -12,7 +12,7 @@ from fs.path import abspath, join, normpath -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import List, Optional, Text, TextIO, Tuple from .base import FS from .info import Info diff --git a/fs/walk.py b/fs/walk.py index 11dd88c7..3e44537d 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -18,7 +18,7 @@ from .path import combine from .path import normpath -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, Callable, diff --git a/fs/wildcard.py b/fs/wildcard.py index b9f58591..6c710cad 100644 --- a/fs/wildcard.py +++ b/fs/wildcard.py @@ -9,10 +9,9 @@ from functools import partial from .lrucache import LRUCache -from . import path -if False: # typing.TYPE_CHECKING - from typing import Callable, Iterable, MutableMapping, Text, Tuple, Pattern +if typing.TYPE_CHECKING: + from typing import Callable, Iterable, Text, Tuple, Pattern _PATTERN_CACHE = LRUCache(1000) # type: LRUCache[Tuple[Text, bool], Pattern] diff --git a/fs/wrap.py b/fs/wrap.py index d8aa7054..7026bcbc 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -24,7 +24,7 @@ from .info import Info from .mode import check_writable -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from datetime import datetime from typing import ( Any, @@ -37,8 +37,8 @@ Text, Tuple, ) - from .base import FS - from .info import Info, RawInfo + from .base import FS # noqa: F401 + from .info import RawInfo from .subfs import SubFS from .permissions import Permissions diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 6565a4ce..c09e9cf3 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals -import copy import typing import six @@ -16,7 +15,7 @@ from .path import abspath, normpath from .error_tools import unwrap_errors -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from datetime import datetime from threading import RLock from typing import ( @@ -25,7 +24,6 @@ BinaryIO, Callable, Collection, - Dict, Iterator, Iterable, IO, @@ -33,7 +31,6 @@ Mapping, Optional, Text, - TextIO, Tuple, Union, ) diff --git a/fs/zipfs.py b/fs/zipfs.py index c347731c..1eefcf6b 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -24,7 +24,7 @@ from .wrapfs import WrapFS from ._url_tools import url_quote -if False: # typing.TYPE_CHECKING +if typing.TYPE_CHECKING: from typing import ( Any, BinaryIO, @@ -181,7 +181,7 @@ def __new__( else: return ReadZipFS(file, encoding=encoding) - if False: # typing.TYPE_CHECKING + if typing.TYPE_CHECKING: def __init__( self, diff --git a/setup.cfg b/setup.cfg index 230c6017..a207101e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,7 +45,24 @@ exclude_lines = pragma: no cover if False: @typing.overload + @overload [tool:pytest] markers = slow: marks tests as slow (deselect with '-m "not slow"') + +[flake8] +extend-ignore = E203,E402,W503 +max-line-length = 88 +per-file-ignores = + fs/__init__.py:F401 + fs/*/__init__.py:F401 + tests/*:E501 + fs/opener/*:F811 + fs/_fscompat.py:F401 + +[isort] +default_section = THIRD_PARTY +known_first_party = fs +known_standard_library = typing +line_length = 88 diff --git a/tests/test_archives.py b/tests/test_archives.py index c0bfff3b..0740e455 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -10,7 +10,6 @@ from fs.enums import ResourceType from fs import walk from fs import errors -from fs.memoryfs import MemoryFS from fs.test import UNICODE_TEXT diff --git a/tests/test_copy.py b/tests/test_copy.py index 160820fa..63e550e9 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -8,8 +8,6 @@ import shutil import calendar -from six import PY2 - import fs.copy from fs import open_fs @@ -356,7 +354,7 @@ def test_copy_dir_if_newer_same_fs(self): src_file1 = self._touch(src_dir, "src" + os.sep + "file1.txt") self._write_file(src_file1) - dst_dir = self._create_sandbox_dir(home=src_dir) + self._create_sandbox_dir(home=src_dir) src_fs = open_fs("osfs://" + src_dir) diff --git a/tests/test_errors.py b/tests/test_errors.py index 0b78fd15..1ed98c54 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -56,7 +56,7 @@ def test_catch_all(self): def test(x): raise errors[x] - for index, exc in enumerate(errors): + for index, _exc in enumerate(errors): try: test(index) except Exception as e: diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 139bc059..7207e9e9 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -124,7 +124,9 @@ def test_manager_with_host(self): with self.assertRaises(errors.RemoteConnectionError) as err_info: with ftp_errors(mem_fs): raise socket.error - self.assertEqual(str(err_info.exception), "unable to connect to ftp.example.com") + self.assertEqual( + str(err_info.exception), "unable to connect to ftp.example.com" + ) @pytest.mark.slow @@ -178,20 +180,34 @@ def tearDown(self): super(TestFTPFS, self).tearDown() def test_ftp_url(self): - self.assertEqual(self.fs.ftp_url, "ftp://{}:{}@{}:{}".format(self.user, self.pasw, self.server.host, self.server.port)) + self.assertEqual( + self.fs.ftp_url, + "ftp://{}:{}@{}:{}".format( + self.user, self.pasw, self.server.host, self.server.port + ), + ) def test_geturl(self): self.fs.makedir("foo") self.fs.create("bar") self.fs.create("foo/bar") self.assertEqual( - self.fs.geturl('foo'), "ftp://{}:{}@{}:{}/foo".format(self.user, self.pasw, self.server.host, self.server.port) + self.fs.geturl("foo"), + "ftp://{}:{}@{}:{}/foo".format( + self.user, self.pasw, self.server.host, self.server.port + ), ) self.assertEqual( - self.fs.geturl('bar'), "ftp://{}:{}@{}:{}/bar".format(self.user, self.pasw, self.server.host, self.server.port) + self.fs.geturl("bar"), + "ftp://{}:{}@{}:{}/bar".format( + self.user, self.pasw, self.server.host, self.server.port + ), ) self.assertEqual( - self.fs.geturl('foo/bar'), "ftp://{}:{}@{}:{}/foo/bar".format(self.user, self.pasw, self.server.host, self.server.port) + self.fs.geturl("foo/bar"), + "ftp://{}:{}@{}:{}/foo/bar".format( + self.user, self.pasw, self.server.host, self.server.port + ), ) def test_host(self): @@ -299,11 +315,7 @@ def tearDownClass(cls): super(TestAnonFTPFS, cls).tearDownClass() def make_fs(self): - return open_fs( - "ftp://{}:{}".format( - self.server.host, self.server.port - ) - ) + return open_fs("ftp://{}:{}".format(self.server.host, self.server.port)) def tearDown(self): shutil.rmtree(self._temp_path) @@ -311,12 +323,23 @@ def tearDown(self): super(TestAnonFTPFS, self).tearDown() def test_ftp_url(self): - self.assertEqual(self.fs.ftp_url, "ftp://{}:{}".format(self.server.host, self.server.port)) + self.assertEqual( + self.fs.ftp_url, "ftp://{}:{}".format(self.server.host, self.server.port) + ) def test_geturl(self): self.fs.makedir("foo") self.fs.create("bar") self.fs.create("foo/bar") - self.assertEqual(self.fs.geturl('foo'), "ftp://{}:{}/foo".format(self.server.host, self.server.port)) - self.assertEqual(self.fs.geturl('bar'), "ftp://{}:{}/bar".format(self.server.host, self.server.port)) - self.assertEqual(self.fs.geturl('foo/bar'), "ftp://{}:{}/foo/bar".format(self.server.host, self.server.port)) + self.assertEqual( + self.fs.geturl("foo"), + "ftp://{}:{}/foo".format(self.server.host, self.server.port), + ) + self.assertEqual( + self.fs.geturl("bar"), + "ftp://{}:{}/bar".format(self.server.host, self.server.port), + ) + self.assertEqual( + self.fs.geturl("foo/bar"), + "ftp://{}:{}/foo/bar".format(self.server.host, self.server.port), + ) diff --git a/tests/test_opener.py b/tests/test_opener.py index 11bc26a5..fc450751 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -29,7 +29,7 @@ def test_registry_repr(self): def test_parse_not_url(self): with self.assertRaises(errors.ParseError): - parsed = opener.parse("foo/bar") + opener.parse("foo/bar") def test_parse_simple(self): parsed = opener.parse("osfs://foo/bar") diff --git a/tests/test_path.py b/tests/test_path.py index d57c278a..e5969d29 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -1,4 +1,4 @@ -from __future__ import unicode_literals, print_function +from __future__ import absolute_import, unicode_literals, print_function """ fstests.test_path: testcases for the fs path functions @@ -8,7 +8,29 @@ import unittest -from fs.path import * +from fs.path import ( + abspath, + basename, + combine, + dirname, + forcedir, + frombase, + isabs, + isbase, + isdotfile, + isparent, + issamedir, + iswildcard, + iteratepath, + join, + normpath, + parts, + recursepath, + relativefrom, + relpath, + split, + splitext, +) class TestPathFunctions(unittest.TestCase): diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index c3570bdb..a90dc0ea 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -163,13 +163,13 @@ def remove_archive(self): def test_read_from_fileobject(self): try: tarfs.TarFS(open(self._temp_path, "rb")) - except: + except Exception: self.fail("Couldn't open tarfs from fileobject") def test_read_from_filename(self): try: tarfs.TarFS(self._temp_path) - except: + except Exception: self.fail("Couldn't open tarfs from filename") def test_read_non_existent_file(self): diff --git a/tests/test_tree.py b/tests/test_tree.py index 805ae708..2a4f942c 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,3 +1,4 @@ +from __future__ import print_function from __future__ import unicode_literals import io diff --git a/tox.ini b/tox.ini index f5a803a9..4a848442 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py34,py35,py36,py37}{,-scandir},pypy,typecheck +envlist = {py27,py34,py35,py36,py37}{,-scandir},pypy,typecheck,lint sitepackages = False skip_missing_interpreters=True @@ -14,3 +14,14 @@ deps = -r {toxinidir}/testrequirements.txt commands = make typecheck whitelist_externals = make + +[testenv:lint] +python = python37 +deps = + flake8 + # flake8-builtins + flake8-bugbear + flake8-comprehensions + # flake8-isort + flake8-mutable +commands = flake8 fs tests From ea2051cc405b12080bd917961e78b4b164a1d34a Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 7 Sep 2019 15:56:42 +0100 Subject: [PATCH 017/309] version bump --- CHANGELOG.md | 2 +- fs/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f423301e..04d2a56b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.11] - Unreleased +## [2.4.11] - 2019-09-07git dif ### Added diff --git a/fs/_version.py b/fs/_version.py index 207d7928..0e4134f2 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.11a0" +__version__ = "2.4.11" From c7aaf7a1ead97c862574eed50387c03e60c89fbc Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 7 Sep 2019 15:58:03 +0100 Subject: [PATCH 018/309] remove garbage from readme --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d2a56b..25e4c1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.11] - 2019-09-07git dif +## [2.4.11] - 2019-09-07 ### Added From 3c3c45c1c9ef07fbbf641af2dc013c2e682b6aa2 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 7 Sep 2019 16:11:12 +0100 Subject: [PATCH 019/309] fix test deprecation warnings --- fs/test.py | 2 +- tests/test_base.py | 4 ++-- tests/test_iotools.py | 2 +- tests/test_path.py | 22 +++++++++++----------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fs/test.py b/fs/test.py index 146938a8..49a0c40f 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1061,7 +1061,7 @@ def test_remove(self): self.fs.makedirs("foo/bar/baz/") error_msg = "resource 'foo/bar/egg/test.txt' not found" - with self.assertRaisesRegexp(errors.ResourceNotFound, error_msg): + with self.assertRaisesRegex(errors.ResourceNotFound, error_msg): self.fs.remove("foo/bar/egg/test.txt") def test_removedir(self): diff --git a/tests/test_base.py b/tests/test_base.py index 188fa68d..937db27d 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -8,7 +8,7 @@ from fs import errors -class TestFS(FS): +class DummyFS(FS): def getinfo(self, path, namespaces=None): pass @@ -33,7 +33,7 @@ def setinfo(self, path, info): class TestBase(unittest.TestCase): def setUp(self): - self.fs = TestFS() + self.fs = DummyFS() def test_validatepath(self): """Test validatepath method.""" diff --git a/tests/test_iotools.py b/tests/test_iotools.py index ffcc2949..afb07d3c 100644 --- a/tests/test_iotools.py +++ b/tests/test_iotools.py @@ -26,7 +26,7 @@ def test_make_stream(self): with self.fs.openbin("foo.bin") as f: data = f.read() - self.assert_(isinstance(data, bytes)) + self.assertTrue(isinstance(data, bytes)) with self.fs.openbin("text.txt", "wb") as f: f.write(UNICODE_TEXT.encode("utf-8")) diff --git a/tests/test_path.py b/tests/test_path.py index e5969d29..52673f03 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -152,14 +152,14 @@ def test_splitext(self): self.assertEqual(splitext(".foo"), (".foo", "")) def test_recursepath(self): - self.assertEquals(recursepath("/"), ["/"]) - self.assertEquals(recursepath("hello"), ["/", "/hello"]) - self.assertEquals(recursepath("/hello/world/"), ["/", "/hello", "/hello/world"]) - self.assertEquals( + self.assertEqual(recursepath("/"), ["/"]) + self.assertEqual(recursepath("hello"), ["/", "/hello"]) + self.assertEqual(recursepath("/hello/world/"), ["/", "/hello", "/hello/world"]) + self.assertEqual( recursepath("/hello/world/", reverse=True), ["/hello/world", "/hello", "/"] ) - self.assertEquals(recursepath("hello", reverse=True), ["/hello", "/"]) - self.assertEquals(recursepath("", reverse=True), ["/"]) + self.assertEqual(recursepath("hello", reverse=True), ["/hello", "/"]) + self.assertEqual(recursepath("", reverse=True), ["/"]) def test_isbase(self): self.assertTrue(isbase("foo", "foo/bar")) @@ -178,7 +178,7 @@ def test_issamedir(self): def test_isdotfile(self): for path in [".foo", ".svn", "foo/.svn", "foo/bar/.svn", "/foo/.bar"]: - self.assert_(isdotfile(path)) + self.assertTrue(isdotfile(path)) for path in ["asfoo", "df.svn", "foo/er.svn", "foo/bar/test.txt", "/foo/bar"]: self.assertFalse(isdotfile(path)) @@ -201,10 +201,10 @@ def test_basename(self): self.assertEqual(basename(path), test_basename) def test_iswildcard(self): - self.assert_(iswildcard("*")) - self.assert_(iswildcard("*.jpg")) - self.assert_(iswildcard("foo/*")) - self.assert_(iswildcard("foo/{}")) + self.assertTrue(iswildcard("*")) + self.assertTrue(iswildcard("*.jpg")) + self.assertTrue(iswildcard("foo/*")) + self.assertTrue(iswildcard("foo/{}")) self.assertFalse(iswildcard("foo")) self.assertFalse(iswildcard("img.jpg")) self.assertFalse(iswildcard("foo/bar")) From 78ad403854169e1eab07d022565ff453dde15006 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 7 Sep 2019 16:20:34 +0100 Subject: [PATCH 020/309] py27 has different method --- fs/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/test.py b/fs/test.py index 49a0c40f..f74d85f3 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1061,7 +1061,8 @@ def test_remove(self): self.fs.makedirs("foo/bar/baz/") error_msg = "resource 'foo/bar/egg/test.txt' not found" - with self.assertRaisesRegex(errors.ResourceNotFound, error_msg): + assertRaisesRegex = getattr(self, "assertRaisesRegex", self.assertRaisesRegexp) + with assertRaisesRegex(errors.ResourceNotFound, error_msg): self.fs.remove("foo/bar/egg/test.txt") def test_removedir(self): From 87ae9f20561af0d341e3cc4f9da2bfd7c9a4449c Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 25 Sep 2019 10:10:30 +0100 Subject: [PATCH 021/309] Corrected docs for forcedir --- fs/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/path.py b/fs/path.py index 49f04bd6..9d6496b5 100644 --- a/fs/path.py +++ b/fs/path.py @@ -509,7 +509,7 @@ def forcedir(path): >>> forcedir("foo/bar/") 'foo/bar/' >>> forcedir("foo/spam.txt") - 'foo/spam.txt' + 'foo/spam.txt/' """ if not path.endswith("/"): From 167ed3c2b8509ef5c6125c083c925d4908e24d10 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 10 Oct 2019 08:45:12 +0200 Subject: [PATCH 022/309] Use `unittest.SkipTest` instead of `pytest.skip` in `fs.test` (#354) --- fs/test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/test.py b/fs/test.py index f74d85f3..53ed290e 100644 --- a/fs/test.py +++ b/fs/test.py @@ -15,8 +15,7 @@ import math import os import time - -import pytest +import unittest import fs.copy import fs.move @@ -1797,7 +1796,7 @@ def test_tree(self): def test_unicode_path(self): if not self.fs.getmeta().get("unicode_paths", False): - return pytest.skip("the filesystem does not support unicode paths.") + raise unittest.SkipTest("the filesystem does not support unicode paths.") self.fs.makedir("földér") self.fs.writetext("☭.txt", "Smells like communism.") @@ -1820,10 +1819,10 @@ def test_unicode_path(self): def test_case_sensitive(self): meta = self.fs.getmeta() if "case_insensitive" not in meta: - return pytest.skip("case sensitivity not known") + raise unittest.SkipTest("case sensitivity not known") if meta.get("case_insensitive", False): - return pytest.skip("the filesystem is not case sensitive.") + raise unittest.SkipTest("the filesystem is not case sensitive.") self.fs.makedir("foo") self.fs.makedir("Foo") From e7778281f411fbb1d47ff8d5c399307a7b59b019 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 3 Nov 2019 11:12:18 +0000 Subject: [PATCH 023/309] silence some mypy errors --- fs/__init__.py | 2 +- fs/opener/__init__.py | 2 +- fs/tarfs.py | 2 +- fs/zipfs.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/__init__.py b/fs/__init__.py index b60a61e7..885141e3 100644 --- a/fs/__init__.py +++ b/fs/__init__.py @@ -1,7 +1,7 @@ """Python filesystem abstraction layer. """ -__import__("pkg_resources").declare_namespace(__name__) +__import__("pkg_resources").declare_namespace(__name__) # type: ignore from ._version import __version__ from .enums import ResourceType, Seek diff --git a/fs/opener/__init__.py b/fs/opener/__init__.py index 91870e7a..c20545eb 100644 --- a/fs/opener/__init__.py +++ b/fs/opener/__init__.py @@ -3,7 +3,7 @@ """ # Declare fs.opener as a namespace package -__import__("pkg_resources").declare_namespace(__name__) +__import__("pkg_resources").declare_namespace(__name__) # type: ignore # Import objects into fs.opener namespace from .base import Opener diff --git a/fs/tarfs.py b/fs/tarfs.py index 20073c9d..4f48d821 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -112,7 +112,7 @@ class TarFS(WrapFS): "gz": (".tar.gz", ".tgz"), } - def __new__( + def __new__( # type: ignore cls, file, # type: Union[Text, BinaryIO] write=False, # type: bool diff --git a/fs/zipfs.py b/fs/zipfs.py index 1eefcf6b..8feb9e56 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -163,7 +163,8 @@ class ZipFS(WrapFS): """ - def __new__( + # TODO: __new__ returning different types may be too 'magical' + def __new__( # type: ignore cls, file, # type: Union[Text, BinaryIO] write=False, # type: bool From d245d72bf0808e50e14eee8df220159b115eb67f Mon Sep 17 00:00:00 2001 From: Diego Argueta Date: Sun, 3 Nov 2019 03:16:05 -0800 Subject: [PATCH 024/309] Start testing PyPy (#357) * Start testing PyPy * Update changelog --- .travis.yml | 10 +++++++--- CHANGELOG.md | 8 ++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d584b1e..1a2370af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,8 @@ python: - "3.6" - "3.7" - "3.8-dev" -# TODO (dargueta): Enable these once we figure out FTP timing issues in PyPy tests -# - 'pypy' -# - 'pypy3.5-7.0' + - 'pypy' + - 'pypy3.5-7.0' # Need 7.0+ due to a bug in earlier versions that broke our tests. matrix: include: @@ -22,6 +21,11 @@ matrix: python: '3.7' env: TOXENV=lint + # Temporary bandaid for https://github.com/PyFilesystem/pyfilesystem2/issues/342 + allow_failures: + - python: pypy + - python: pypy3.5-7.0 + before_install: - pip install -U tox tox-travis - pip --version diff --git a/CHANGELOG.md b/CHANGELOG.md index 25e4c1c3..7126b0a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [2.4.12] - (Unreleased) + +### Changed + +- Start testing on PyPy. Due to [#342](https://github.com/PyFilesystem/pyfilesystem2/issues/342) + we have to treat PyPy builds specially and allow them to fail, but at least we'll + be able to see if we break something aside from known issues with FTP tests. + ## [2.4.11] - 2019-09-07 ### Added From 58fdafd6d3a5b6045aa71b0581498310a577c023 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 3 Nov 2019 11:27:10 +0000 Subject: [PATCH 025/309] fix type error, pin mypy --- fs/tools.py | 6 ++++-- tox.ini | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fs/tools.py b/fs/tools.py index 8a16d289..7dac4d25 100644 --- a/fs/tools.py +++ b/fs/tools.py @@ -15,7 +15,7 @@ from .path import recursepath if typing.TYPE_CHECKING: - from typing import IO, List, Optional, Text + from typing import IO, List, Optional, Text, Union from .base import FS @@ -52,7 +52,9 @@ def copy_file_data(src_file, dst_file, chunk_size=None): read = src_file.read write = dst_file.write # The 'or None' is so that it works with binary and text files - for chunk in iter(lambda: read(_chunk_size) or None, None): + for chunk in iter( + lambda: read(_chunk_size) or None, None + ): # type: Optional[Union[bytes, str]] write(chunk) diff --git a/tox.ini b/tox.ini index 4a848442..887ecd56 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ commands = coverage run -m pytest --cov-append {posargs} {toxinidir}/tests [testenv:typecheck] python = python37 deps = - mypy + mypy==0.740 -r {toxinidir}/testrequirements.txt commands = make typecheck whitelist_externals = make From 129567606066cd002bb45a11aae543f7b73f0134 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 3 Nov 2019 11:29:53 +0000 Subject: [PATCH 026/309] use non-dev py3.8 --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1a2370af..72437a67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,17 +8,17 @@ python: - "3.5" - "3.6" - "3.7" - - "3.8-dev" - - 'pypy' - - 'pypy3.5-7.0' # Need 7.0+ due to a bug in earlier versions that broke our tests. + - "3.8" + - "pypy" + - "pypy3.5-7.0" # Need 7.0+ due to a bug in earlier versions that broke our tests. matrix: include: - name: "Type checking" python: "3.7" env: TOXENV=typecheck - - name: 'Lint' - python: '3.7' + - name: "Lint" + python: "3.7" env: TOXENV=lint # Temporary bandaid for https://github.com/PyFilesystem/pyfilesystem2/issues/342 From 24e74e01018a70fd9866f27e51300edda9ef633b Mon Sep 17 00:00:00 2001 From: Alex Povel <48824213+alexpovel@users.noreply.github.com> Date: Thu, 20 Feb 2020 13:32:45 +0100 Subject: [PATCH 027/309] Adjust docstring to match function A little *f* got lost --- fs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index a4b7aefb..a6218b93 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1409,7 +1409,7 @@ def writetext( path (str): Destination path on the filesystem. contents (str): Text to be written. encoding (str, optional): Encoding of destination file - (defaults to ``'ut-8'``). + (defaults to ``'utf-8'``). errors (str, optional): How encoding errors should be treated (same as `io.open`). newline (str): Newline parameter (same as `io.open`). From 6fc6ac5174e69b7fee5828114692eb250b057057 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Thu, 2 Apr 2020 18:11:28 +1100 Subject: [PATCH 028/309] Fix simple typo: recusrive -> recursive (#366) Closes #365 --- docs/source/globbing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/globbing.rst b/docs/source/globbing.rst index c72392fc..ac81c185 100644 --- a/docs/source/globbing.rst +++ b/docs/source/globbing.rst @@ -15,7 +15,7 @@ Matching Files and Directories In a glob pattern, A ``*`` means match anything text in a filename. A ``?`` matches any single character. A ``**`` matches any number of subdirectories, -making the glob *recusrive*. If the glob pattern ends in a ``/``, it will +making the glob *recursive*. If the glob pattern ends in a ``/``, it will only match directory paths, otherwise it will match files and directories. .. note:: From f120347790fc1689cac33076d8ad0bcf0cbfd63a Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 16 May 2020 14:52:29 +0100 Subject: [PATCH 029/309] Enable sponsorship --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..4abae047 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: willmcgugan From b4b93d56117cf36ea71fb2c8d6bc5d22f8d85a92 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 16 May 2020 15:04:21 +0100 Subject: [PATCH 030/309] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 4abae047..b10616c2 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -github: willmcgugan +github: https://www.willmcgugan.com/sponsorship/ From 8e178910ad9c0296610c2d5e6fcada13053a3b46 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sat, 16 May 2020 15:04:48 +0100 Subject: [PATCH 031/309] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index b10616c2..c73fbbd2 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -github: https://www.willmcgugan.com/sponsorship/ +custom: https://www.willmcgugan.com/sponsorship/ From 63a14f7221a8793de5a1324d067beb2bab7febb3 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Sun, 17 May 2020 12:51:31 +0100 Subject: [PATCH 032/309] Update FUNDING.yml --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index c73fbbd2..6762ecbc 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,3 @@ # These are supported funding model platforms -custom: https://www.willmcgugan.com/sponsorship/ +custom: willmcgugan From af3002ad8ec70de015037f1bfe2065c10c7a797f Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Thu, 10 Sep 2020 11:59:36 +0200 Subject: [PATCH 033/309] Include additional files in source distributions, fixes #364 This will ensure the missing conftest.py file is included, as well as the docs folder. --- CHANGELOG.md | 2 ++ CONTRIBUTORS.md | 1 + MANIFEST.in | 4 ++++ 3 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7126b0a9..d3648caa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Start testing on PyPy. Due to [#342](https://github.com/PyFilesystem/pyfilesystem2/issues/342) we have to treat PyPy builds specially and allow them to fail, but at least we'll be able to see if we break something aside from known issues with FTP tests. +- Include docs in source distributions as well the whole tests folder, + ensuring `conftest.py` is present, fixes [#364](https://github.com/PyFilesystem/pyfilesystem2/issues/364). ## [2.4.11] - 2019-09-07 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cb55cf74..432bf23a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,6 +6,7 @@ Many thanks to the following developers for contributing to this project: - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) - [Giampaolo](https://github.com/gpcimino) +- [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) diff --git a/MANIFEST.in b/MANIFEST.in index 1aba38f6..99db56b1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,5 @@ include LICENSE +graft tests +graft docs +global-exclude __pycache__ +global-exclude *.py[co] From 5df7f7de0739748568622792894df21acefef36c Mon Sep 17 00:00:00 2001 From: Louis Sautier Date: Sun, 13 Sep 2020 16:27:31 +0200 Subject: [PATCH 034/309] Do not test the patched copy implementation with Python 3.8+, fixes #421 Also stop patching copy with Python 3.8, reverting part of 6f89b81daab7b604a77eff232639b03b6fc55ab2. --- CHANGELOG.md | 1 + CONTRIBUTORS.md | 1 + fs/osfs.py | 2 +- tests/test_osfs.py | 6 ++++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7126b0a9..357fa738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Start testing on PyPy. Due to [#342](https://github.com/PyFilesystem/pyfilesystem2/issues/342) we have to treat PyPy builds specially and allow them to fail, but at least we'll be able to see if we break something aside from known issues with FTP tests. +- Stop patching copy with Python 3.8+ because it already uses sendfile. ## [2.4.11] - 2019-09-07 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cb55cf74..432bf23a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,6 +6,7 @@ Many thanks to the following developers for contributing to this project: - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) - [Giampaolo](https://github.com/gpcimino) +- [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) diff --git a/fs/osfs.py b/fs/osfs.py index ec68d0ad..f854b16a 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -419,7 +419,7 @@ def _check_copy(self, src_path, dst_path, overwrite=False): raise errors.DirectoryExpected(dirname(dst_path)) return _src_path, _dst_path - if sys.version_info[:2] <= (3, 8) and sendfile is not None: + if sys.version_info[:2] < (3, 8) and sendfile is not None: _sendfile_error_codes = { errno.EIO, diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 18eabd58..f656646c 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -6,6 +6,7 @@ import os import shutil import tempfile +import sys import unittest import pytest @@ -88,6 +89,11 @@ def test_expand_vars(self): self.assertNotIn("TYRIONLANISTER", fs2.getsyspath("/")) @pytest.mark.skipif(osfs.sendfile is None, reason="sendfile not supported") + @pytest.mark.skipif( + sys.version_info >= (3, 8), + reason="the copy function uses sendfile in Python 3.8+, " + "making the patched implementation irrelevant", + ) def test_copy_sendfile(self): # try copying using sendfile with mock.patch.object(osfs, "sendfile") as sendfile: From 34084cf5d1027d413fff265c93af5b8dc0685ea7 Mon Sep 17 00:00:00 2001 From: jcharlong <58079248+jcharlong@users.noreply.github.com> Date: Fri, 25 Sep 2020 05:54:13 -0400 Subject: [PATCH 035/309] Fix None docstring when running in -OO mode (#414) --- CHANGELOG.md | 1 + CONTRIBUTORS.md | 1 + fs/base.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7597621a..0449d14f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Include docs in source distributions as well the whole tests folder, ensuring `conftest.py` is present, fixes [#364](https://github.com/PyFilesystem/pyfilesystem2/issues/364). - Stop patching copy with Python 3.8+ because it already uses sendfile. +- Fixed crash when CPython's -OO flag is used ## [2.4.11] - 2019-09-07 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 432bf23a..de54b668 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,6 +6,7 @@ Many thanks to the following developers for contributing to this project: - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) - [Giampaolo](https://github.com/gpcimino) +- [Justin Charlong](https://github.com/jcharlong) - [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) - [Will McGugan](https://github.com/willmcgugan) diff --git a/fs/base.py b/fs/base.py index a6218b93..5eb0b90b 100644 --- a/fs/base.py +++ b/fs/base.py @@ -84,7 +84,7 @@ def _method(*args, **kwargs): """.format( method.__name__ ) - if hasattr(_method, "__doc__"): + if getattr(_method, "__doc__", None) is not None: _method.__doc__ += deprecated_msg return _method From a2aa9f1336fc0c8b899a9fb9b8ae7ab6695b26e5 Mon Sep 17 00:00:00 2001 From: Morten Engelhardt Olsen Date: Wed, 30 Sep 2020 10:06:25 -0700 Subject: [PATCH 036/309] Use a non-greedy pattern when capturing mtime in FTP (#395) By making this capture pattern non-greedy it will no longer capture until the last AM or PM in a directory listing line. --- CHANGELOG.md | 1 + CONTRIBUTORS.md | 1 + fs/_ftp_parse.py | 2 +- tests/test_ftp_parse.py | 8 ++++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0449d14f..7cec240a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ensuring `conftest.py` is present, fixes [#364](https://github.com/PyFilesystem/pyfilesystem2/issues/364). - Stop patching copy with Python 3.8+ because it already uses sendfile. - Fixed crash when CPython's -OO flag is used +- Fixed error when parsing timestamps from a FTP directory served from a WindowsNT FTP Server, fixes [#395](https://github.com/PyFilesystem/pyfilesystem2/issues/395). ## [2.4.11] - 2019-09-07 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index de54b668..3b3e8c2d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -11,3 +11,4 @@ Many thanks to the following developers for contributing to this project: - [Martin Larralde](https://github.com/althonos) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) +- [Morten Engelhardt Olsen](https://github.com/xoriath) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index b50f75eb..b503d737 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -41,7 +41,7 @@ RE_WINDOWSNT = re.compile( r""" ^ - (?P.*(AM|PM)) + (?P.*?(AM|PM)) \s* (?P(|\d*)) \s* diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index c649a1f4..d0abc05a 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -166,6 +166,7 @@ def test_decode_windowsnt(self, mock_localtime): directory = """\ 11-02-17 02:00AM docs 11-02-17 02:12PM images +11-02-17 02:12PM AM to PM 11-02-17 03:33PM 9276 logo.gif """ expected = [ @@ -183,6 +184,13 @@ def test_decode_windowsnt(self, mock_localtime): "ls": "11-02-17 02:12PM images" }, }, + { + "basic": {"is_dir": True, "name": "AM to PM"}, + "details": {"modified": 1486822320.0, "type": 1}, + "ftp": { + "ls": "11-02-17 02:12PM AM to PM" + }, + }, { "basic": {"is_dir": False, "name": "logo.gif"}, "details": {"modified": 1486827180.0, "size": 9276, "type": 2}, From 9a8f94a71d919c50efd2c63af6437bbd35cf71f0 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 13 Oct 2020 14:20:20 +0200 Subject: [PATCH 037/309] Fix documentation issues for release v2.4.12 (#434) * Fix missing `unittest` import in documentation example * Fix description in docstring of `Mode.to_platform_bin` * Update `CHANGELOG.md` with fixes and typo correction --- CHANGELOG.md | 10 ++++++++-- docs/source/implementers.rst | 3 ++- fs/mode.py | 3 +-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cec240a..562645b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Start testing on PyPy. Due to [#342](https://github.com/PyFilesystem/pyfilesystem2/issues/342) we have to treat PyPy builds specially and allow them to fail, but at least we'll be able to see if we break something aside from known issues with FTP tests. -- Include docs in source distributions as well the whole tests folder, +- Include docs in source distributions as well as the whole tests folder, ensuring `conftest.py` is present, fixes [#364](https://github.com/PyFilesystem/pyfilesystem2/issues/364). -- Stop patching copy with Python 3.8+ because it already uses sendfile. +- Stop patching copy with Python 3.8+ because it already uses `sendfile`. + +### Fixed + - Fixed crash when CPython's -OO flag is used - Fixed error when parsing timestamps from a FTP directory served from a WindowsNT FTP Server, fixes [#395](https://github.com/PyFilesystem/pyfilesystem2/issues/395). +- Fixed documentation of `Mode.to_platform_bin`. Closes [#382](https://github.com/PyFilesystem/pyfilesystem2/issues/382). +- Fixed the code example in the "Testing Filesystems" section of the + "Implementing Filesystems" guide. Closes [#407](https://github.com/PyFilesystem/pyfilesystem2/issues/407). ## [2.4.11] - 2019-09-07 diff --git a/docs/source/implementers.rst b/docs/source/implementers.rst index 51c33891..bb055d69 100644 --- a/docs/source/implementers.rst +++ b/docs/source/implementers.rst @@ -40,9 +40,10 @@ To test your implementation, you can borrow the test suite used to test the buil Here's the simplest possible example to test a filesystem class called ``MyFS``:: + import unittest from fs.test import FSTestCases - class TestMyFS(FSTestCases): + class TestMyFS(FSTestCases, unittest.TestCase): def make_fs(self): # Return an instance of your FS object here diff --git a/fs/mode.py b/fs/mode.py index 5e8c795d..16d51875 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -84,8 +84,7 @@ def to_platform_bin(self): # type: () -> Text """Get a *binary* mode string for the current platform. - Currently, this just removes the 'x' on PY2 because PY2 doesn't - support exclusive mode. + This removes the 't' and adds a 'b' if needed. """ _mode = self.to_platform().replace("t", "") From 526438d9aae52eb7733b04ffec254ef023902323 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 13 Oct 2020 15:14:01 +0200 Subject: [PATCH 038/309] Require `FS.openbin` to return a file-handle in binary mode (#435) * Add test to ensure mode of files opened with `openbin` contain `b` flag * Fix `FTPFS.openbin` not implicitly converting mode to binary mode * Add missing `mode` attribute to `_MemoryFile` objects * Add missing type annotation to `_MemoryFile.mode` property --- CHANGELOG.md | 6 ++++++ fs/ftpfs.py | 2 +- fs/memoryfs.py | 5 +++++ fs/test.py | 4 ++++ 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 562645b3..fa072e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [2.4.12] - (Unreleased) +### Added + +- Missing `mode` attribute to `_MemoryFile` objects returned by `MemoryFS.openbin`. + ### Changed - Start testing on PyPy. Due to [#342](https://github.com/PyFilesystem/pyfilesystem2/issues/342) @@ -23,6 +27,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed documentation of `Mode.to_platform_bin`. Closes [#382](https://github.com/PyFilesystem/pyfilesystem2/issues/382). - Fixed the code example in the "Testing Filesystems" section of the "Implementing Filesystems" guide. Closes [#407](https://github.com/PyFilesystem/pyfilesystem2/issues/407). +- Fixed `FTPFS.openbin` not implicitly opening files in binary mode like expected + from `openbin`. Closes [#406](https://github.com/PyFilesystem/pyfilesystem2/issues/406). ## [2.4.11] - 2019-09-07 diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 11d2c4cc..6442a1ba 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -682,7 +682,7 @@ def openbin(self, path, mode="r", buffering=-1, **options): raise errors.FileExpected(path) if _mode.exclusive: raise errors.FileExists(path) - ftp_file = FTPFile(self, _path, mode) + ftp_file = FTPFile(self, _path, _mode.to_platform_bin()) return ftp_file # type: ignore def remove(self, path): diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 0814b2e0..2a011b21 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -75,6 +75,11 @@ def __str__(self): _template = "" return _template.format(path=self._path, mode=self._mode) + @property + def mode(self): + # type: () -> Text + return self._mode.to_platform_bin() + @contextlib.contextmanager def _seek_lock(self): # type: () -> Iterator[None] diff --git a/fs/test.py b/fs/test.py index 53ed290e..37751731 100644 --- a/fs/test.py +++ b/fs/test.py @@ -774,6 +774,7 @@ def test_openbin_rw(self): with self.fs.openbin("foo/hello", "w") as f: repr(f) + self.assertIn("b", f.mode) self.assertIsInstance(f, io.IOBase) self.assertTrue(f.writable()) self.assertFalse(f.readable()) @@ -787,6 +788,7 @@ def test_openbin_rw(self): # Read it back with self.fs.openbin("foo/hello", "r") as f: + self.assertIn("b", f.mode) self.assertIsInstance(f, io.IOBase) self.assertTrue(f.readable()) self.assertFalse(f.writable()) @@ -927,6 +929,7 @@ def test_openbin(self): with self.fs.openbin("file.bin", "wb") as write_file: repr(write_file) text_type(write_file) + self.assertIn("b", write_file.mode) self.assertIsInstance(write_file, io.IOBase) self.assertTrue(write_file.writable()) self.assertFalse(write_file.readable()) @@ -938,6 +941,7 @@ def test_openbin(self): with self.fs.openbin("file.bin", "rb") as read_file: repr(write_file) text_type(write_file) + self.assertIn("b", read_file.mode) self.assertIsInstance(read_file, io.IOBase) self.assertTrue(read_file.readable()) self.assertFalse(read_file.writable()) From c5d193bf7e4b3696386d6d4439e8192d5060f6d6 Mon Sep 17 00:00:00 2001 From: Nick Henderson Date: Sun, 18 Oct 2020 04:13:46 -0700 Subject: [PATCH 039/309] Add readinto method for MemoryFS and FTPFS file objects (#381) * add _MemoryFile.readinto method * update CHANGELOG.md * call on_access in _MemoryFile next and readlines methods * add myself to CONTRIBUTORS.md * add mode checks for _MemoryFile readline and readlines methods * fix type annotation for _MemoryFile.readinto * add test for readinto method in open file object * add FTPFile.readinto method * use buffer as variable name for bytearray objects * use bytes_read variable in readinto methods --- CHANGELOG.md | 2 ++ CONTRIBUTORS.md | 1 + fs/ftpfs.py | 7 +++++++ fs/iotools.py | 4 ++-- fs/memoryfs.py | 14 ++++++++++++++ fs/test.py | 5 +++++ 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa072e2e..3b965c9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Missing `mode` attribute to `_MemoryFile` objects returned by `MemoryFS.openbin`. +- Missing `readinto` method for `MemoryFS` and `FTPFS` file objects. Closes + [#380](https://github.com/PyFilesystem/pyfilesystem2/issues/380). ### Changed diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3b3e8c2d..ba15fd73 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -9,6 +9,7 @@ Many thanks to the following developers for contributing to this project: - [Justin Charlong](https://github.com/jcharlong) - [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) +- [Nick Henderson](https://github.com/nwh) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) - [Morten Engelhardt Olsen](https://github.com/xoriath) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 6442a1ba..e3a39411 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -235,6 +235,13 @@ def read(self, size=-1): remaining -= len(chunk) return b"".join(chunks) + def readinto(self, buffer): + # type: (bytearray) -> int + data = self.read(len(buffer)) + bytes_read = len(data) + buffer[:bytes_read] = data + return bytes_read + def readline(self, size=-1): # type: (int) -> bytes return next(line_iterator(self, size)) # type: ignore diff --git a/fs/iotools.py b/fs/iotools.py index 26402ff3..44849680 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -117,7 +117,7 @@ def readinto(self, b): except AttributeError: data = self._f.read(len(b)) bytes_read = len(data) - b[: len(data)] = data + b[:bytes_read] = data return bytes_read @typing.no_type_check @@ -128,7 +128,7 @@ def readinto1(self, b): except AttributeError: data = self._f.read1(len(b)) bytes_read = len(data) - b[: len(data)] = data + b[:bytes_read] = data return bytes_read def readline(self, limit=-1): diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 2a011b21..d1c23724 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -113,12 +113,15 @@ def __iter__(self): def next(self): # type: () -> bytes with self._seek_lock(): + self.on_access() return next(self._bytes_io) __next__ = next def readline(self, size=-1): # type: (int) -> bytes + if not self._mode.reading: + raise IOError("File not open for reading") with self._seek_lock(): self.on_access() return self._bytes_io.readline(size) @@ -142,9 +145,20 @@ def readable(self): # type: () -> bool return self._mode.reading + def readinto(self, buffer): + # type (bytearray) -> Optional[int] + if not self._mode.reading: + raise IOError("File not open for reading") + with self._seek_lock(): + self.on_access() + return self._bytes_io.readinto(buffer) + def readlines(self, hint=-1): # type: (int) -> List[bytes] + if not self._mode.reading: + raise IOError("File not open for reading") with self._seek_lock(): + self.on_access() return self._bytes_io.readlines(hint) def seekable(self): diff --git a/fs/test.py b/fs/test.py index 37751731..9d8e7ed7 100644 --- a/fs/test.py +++ b/fs/test.py @@ -870,6 +870,11 @@ def test_open_files(self): self.assertTrue(f.readable()) self.assertFalse(f.closed) self.assertEqual(f.readlines(8), [b"Hello\n", b"World\n"]) + self.assertEqual(f.tell(), 12) + buffer = bytearray(4) + self.assertEqual(f.readinto(buffer), 4) + self.assertEqual(f.tell(), 16) + self.assertEqual(buffer, b"foo\n") with self.assertRaises(IOError): f.write(b"no") self.assertTrue(f.closed) From 19e4387cf9b2bd59fa2e2b7b6f4053287bfab23f Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 5 Nov 2020 08:45:09 +0100 Subject: [PATCH 040/309] Issue #438: Extended parsing compatibility of the FTP `LIST` command for Windows servers. The parsing process now properly supports 24-hour time format, both with and without leading zeros. --- CHANGELOG.md | 2 ++ CONTRIBUTORS.md | 1 + fs/_ftp_parse.py | 35 +++++++++++++++++++++++------- tests/test_ftp_parse.py | 48 +++++++++++++++++++++++++---------------- 4 files changed, 60 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b965c9c..9e0a7680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Missing `mode` attribute to `_MemoryFile` objects returned by `MemoryFS.openbin`. - Missing `readinto` method for `MemoryFS` and `FTPFS` file objects. Closes [#380](https://github.com/PyFilesystem/pyfilesystem2/issues/380). +- Added compatibility if a Windows FTP server returns file information to the + `LIST` command with 24-hour times. Closes [#438](https://github.com/PyFilesystem/pyfilesystem2/issues/438). ### Changed diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index ba15fd73..31d5da55 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -13,3 +13,4 @@ Many thanks to the following developers for contributing to this project: - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) - [Morten Engelhardt Olsen](https://github.com/xoriath) +- [Andreas Tollkötter](https://github.com/atollk) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index b503d737..235a6c16 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -41,14 +41,17 @@ RE_WINDOWSNT = re.compile( r""" ^ - (?P.*?(AM|PM)) - \s* - (?P(|\d*)) - \s* + (?P\S+) + \s+ + (?P\S+(AM|PM)?) + \s+ + (?P(|\d+)) + \s+ (?P.*) $ """, - re.VERBOSE) + re.VERBOSE, +) def get_decoders(): @@ -104,6 +107,10 @@ def _parse_time(t, formats): return epoch_time +def _decode_linux_time(mtime): + return _parse_time(mtime, formats=["%b %d %Y", "%b %d %H:%M"]) + + def decode_linux(line, match): perms, links, uid, gid, size, mtime, name = match.groups() is_link = perms.startswith("l") @@ -114,7 +121,7 @@ def decode_linux(line, match): _link_name = _link_name.strip() permissions = Permissions.parse(perms[1:]) - mtime_epoch = _parse_time(mtime, formats=["%b %d %Y", "%b %d %H:%M"]) + mtime_epoch = _decode_linux_time(mtime) name = unicodedata.normalize("NFC", name) @@ -138,12 +145,22 @@ def decode_linux(line, match): return raw_info +def _decode_windowsnt_time(date, time): + while len(time.split(":")[0]) < 2: + time = "0" + time + return _parse_time( + date + " " + time, formats=["%d-%m-%y %I:%M%p", "%d-%m-%y %H:%M"] + ) + + def decode_windowsnt(line, match): """ - Decodes a Windows NT FTP LIST line like these two: + Decodes a Windows NT FTP LIST line like one of these: `11-02-18 02:12PM images` `11-02-18 03:33PM 9276 logo.gif` + + Alternatively, the time (02:12PM) might also be present in 24-hour format (14:12). """ is_dir = match.group("size") == "" @@ -161,7 +178,9 @@ def decode_windowsnt(line, match): if not is_dir: raw_info["details"]["size"] = int(match.group("size")) - modified = _parse_time(match.group("modified"), formats=["%d-%m-%y %I:%M%p"]) + modified = _decode_windowsnt_time( + match.group("modified_date"), match.group("modified_time") + ) if modified is not None: raw_info["details"]["modified"] = modified diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index d0abc05a..1ee3cf86 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -17,17 +17,18 @@ class TestFTPParse(unittest.TestCase): @mock.patch("time.localtime") def test_parse_time(self, mock_localtime): self.assertEqual( - ftp_parse._parse_time("JUL 05 1974", formats=["%b %d %Y"]), - 142214400.0) + ftp_parse._parse_time("JUL 05 1974", formats=["%b %d %Y"]), 142214400.0 + ) mock_localtime.return_value = time2017 self.assertEqual( - ftp_parse._parse_time("JUL 05 02:00", formats=["%b %d %H:%M"]), - 1499220000.0) + ftp_parse._parse_time("JUL 05 02:00", formats=["%b %d %H:%M"]), 1499220000.0 + ) self.assertEqual( ftp_parse._parse_time("05-07-17 02:00AM", formats=["%d-%m-%y %I:%M%p"]), - 1499220000.0) + 1499220000.0, + ) self.assertEqual(ftp_parse._parse_time("notadate", formats=["%b %d %Y"]), None) @@ -164,39 +165,50 @@ def test_decode_linux(self, mock_localtime): def test_decode_windowsnt(self, mock_localtime): mock_localtime.return_value = time2017 directory = """\ +unparsable line 11-02-17 02:00AM docs 11-02-17 02:12PM images -11-02-17 02:12PM AM to PM +11-02-17 02:12PM AM to PM 11-02-17 03:33PM 9276 logo.gif +05-11-20 22:11 src +11-02-17 01:23 1 12 +11-02-17 4:54 0 icon.bmp """ expected = [ { "basic": {"is_dir": True, "name": "docs"}, "details": {"modified": 1486778400.0, "type": 1}, - "ftp": { - "ls": "11-02-17 02:00AM docs" - }, + "ftp": {"ls": "11-02-17 02:00AM docs"}, }, { "basic": {"is_dir": True, "name": "images"}, "details": {"modified": 1486822320.0, "type": 1}, - "ftp": { - "ls": "11-02-17 02:12PM images" - }, + "ftp": {"ls": "11-02-17 02:12PM images"}, }, { "basic": {"is_dir": True, "name": "AM to PM"}, "details": {"modified": 1486822320.0, "type": 1}, - "ftp": { - "ls": "11-02-17 02:12PM AM to PM" - }, + "ftp": {"ls": "11-02-17 02:12PM AM to PM"}, }, { "basic": {"is_dir": False, "name": "logo.gif"}, "details": {"modified": 1486827180.0, "size": 9276, "type": 2}, - "ftp": { - "ls": "11-02-17 03:33PM 9276 logo.gif" - }, + "ftp": {"ls": "11-02-17 03:33PM 9276 logo.gif"}, + }, + { + "basic": {"is_dir": True, "name": "src"}, + "details": {"modified": 1604614260.0, "type": 1}, + "ftp": {"ls": "05-11-20 22:11 src"}, + }, + { + "basic": {"is_dir": False, "name": "12"}, + "details": {"modified": 1486776180.0, "size": 1, "type": 2}, + "ftp": {"ls": "11-02-17 01:23 1 12"}, + }, + { + "basic": {"is_dir": False, "name": "icon.bmp"}, + "details": {"modified": 1486788840.0, "size": 0, "type": 2}, + "ftp": {"ls": "11-02-17 4:54 0 icon.bmp"}, }, ] From 3bbbc482a3e945b587b9963177003d31e71c6b4a Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 5 Nov 2020 22:24:51 +0100 Subject: [PATCH 041/309] Adding to the previous commit, performed minor changes according to suggestions from a code review. --- fs/_ftp_parse.py | 8 ++++---- tests/test_ftp_parse.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 235a6c16..b235159b 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -145,11 +145,11 @@ def decode_linux(line, match): return raw_info -def _decode_windowsnt_time(date, time): - while len(time.split(":")[0]) < 2: - time = "0" + time +def _decode_windowsnt_time(mdate, mtime): + if len(mtime.split(":")[0]) == 1: + mtime = "0" + mtime return _parse_time( - date + " " + time, formats=["%d-%m-%y %I:%M%p", "%d-%m-%y %H:%M"] + mdate + " " + mtime, formats=["%d-%m-%y %I:%M%p", "%d-%m-%y %H:%M"] ) diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index 1ee3cf86..b9a69cf1 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -173,6 +173,9 @@ def test_decode_windowsnt(self, mock_localtime): 05-11-20 22:11 src 11-02-17 01:23 1 12 11-02-17 4:54 0 icon.bmp +11-02-17 4:54AM 0 icon.gif +11-02-17 4:54PM 0 icon.png +11-02-17 16:54 0 icon.jpg """ expected = [ { @@ -210,6 +213,21 @@ def test_decode_windowsnt(self, mock_localtime): "details": {"modified": 1486788840.0, "size": 0, "type": 2}, "ftp": {"ls": "11-02-17 4:54 0 icon.bmp"}, }, + { + "basic": {"is_dir": False, "name": "icon.gif"}, + "details": {"modified": 1486788840.0, "size": 0, "type": 2}, + "ftp": {"ls": "11-02-17 4:54AM 0 icon.gif"}, + }, + { + "basic": {"is_dir": False, "name": "icon.png"}, + "details": {"modified": 1486832040.0, "size": 0, "type": 2}, + "ftp": {"ls": "11-02-17 4:54PM 0 icon.png"}, + }, + { + "basic": {"is_dir": False, "name": "icon.jpg"}, + "details": {"modified": 1486832040.0, "size": 0, "type": 2}, + "ftp": {"ls": "11-02-17 16:54 0 icon.jpg"}, + }, ] parsed = ftp_parse.parse(directory.splitlines()) From 7b76fb110ac7e22ae369bdfd08a5c48ad75fcd48 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 19 Nov 2020 22:16:33 +0100 Subject: [PATCH 042/309] Sorted CONTRIBUTORS.md alphabetically. --- CONTRIBUTORS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 31d5da55..5df27157 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -2,6 +2,7 @@ Many thanks to the following developers for contributing to this project: +- [Andreas Tollkötter](https://github.com/atollk) - [C. W.](https://github.com/chfw) - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) @@ -9,8 +10,7 @@ Many thanks to the following developers for contributing to this project: - [Justin Charlong](https://github.com/jcharlong) - [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) +- [Morten Engelhardt Olsen](https://github.com/xoriath) - [Nick Henderson](https://github.com/nwh) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) -- [Morten Engelhardt Olsen](https://github.com/xoriath) -- [Andreas Tollkötter](https://github.com/atollk) From d1997bab5100182ac6f6fe5e2c77c69cf2dcd9f7 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 19 Nov 2020 22:26:48 +0100 Subject: [PATCH 043/309] Cleaned up some redundant and unclear code. In fs._ftp_parse._parse_time, a noop-line was removed. On request of code review, the loop to determine the suitable time format was also improved upon. See https://github.com/PyFilesystem/pyfilesystem2/pull/439#issuecomment-728622272 for details. --- fs/_ftp_parse.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index b235159b..759a6dce 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -84,17 +84,21 @@ def parse_line(line): return None -def _parse_time(t, formats): - t = " ".join(token.strip() for token in t.lower().split(" ")) - - _t = None +def _find_suitable_format(t, formats): for frmt in formats: try: - _t = time.strptime(t, frmt) + if time.strptime(t, frmt): + return frmt except ValueError: continue - if not _t: + return None + + +def _parse_time(t, formats): + frmt = _find_suitable_format(t, formats) + if frmt is None: return None + _t = time.strptime(t, frmt) year = _t.tm_year if _t.tm_year != 1900 else time.localtime().tm_year month = _t.tm_mon From 89d09199fea18a95e136dfb38491325ba1da81da Mon Sep 17 00:00:00 2001 From: atollk Date: Fri, 20 Nov 2020 07:33:00 +0100 Subject: [PATCH 044/309] fs._ftp_parse, removed some code that was only assumed to be necessary due to incomplete documentation The standard library function `time.strptime` using the format "%H" was falsely assumed to require a two-digit number (00-23). As it turns out, one-digit numbers (0-9) are also valid, so we don't have to manually prepend a zero. --- fs/_ftp_parse.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 759a6dce..73553f13 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -149,12 +149,8 @@ def decode_linux(line, match): return raw_info -def _decode_windowsnt_time(mdate, mtime): - if len(mtime.split(":")[0]) == 1: - mtime = "0" + mtime - return _parse_time( - mdate + " " + mtime, formats=["%d-%m-%y %I:%M%p", "%d-%m-%y %H:%M"] - ) +def _decode_windowsnt_time(mtime): + return _parse_time(mtime, formats=["%d-%m-%y %I:%M%p", "%d-%m-%y %H:%M"]) def decode_windowsnt(line, match): @@ -183,7 +179,7 @@ def decode_windowsnt(line, match): raw_info["details"]["size"] = int(match.group("size")) modified = _decode_windowsnt_time( - match.group("modified_date"), match.group("modified_time") + match.group("modified_date") + " " + match.group("modified_time") ) if modified is not None: raw_info["details"]["modified"] = modified From c04ff0a4ae43238648c2cd6b80c9b118cc1518a0 Mon Sep 17 00:00:00 2001 From: atollk Date: Mon, 23 Nov 2020 16:25:48 +0100 Subject: [PATCH 045/309] Re-merged two functions in fs._ftp_parse. See https://github.com/PyFilesystem/pyfilesystem2/pull/439#pullrequestreview-535549737 for details. The function `_find_suitable_format` was inlined into `_parse_time`. --- fs/_ftp_parse.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 73553f13..ffbcbc65 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -95,10 +95,16 @@ def _find_suitable_format(t, formats): def _parse_time(t, formats): - frmt = _find_suitable_format(t, formats) - if frmt is None: + _t = None + for frmt in formats: + try: + _t = time.strptime(t, frmt) + if t: + break + except ValueError: + continue + if not _t: return None - _t = time.strptime(t, frmt) year = _t.tm_year if _t.tm_year != 1900 else time.localtime().tm_year month = _t.tm_mon From 4a19bb8b4cafb9324c69ebcbaa737940eb87cdbe Mon Sep 17 00:00:00 2001 From: atollk Date: Mon, 23 Nov 2020 16:27:23 +0100 Subject: [PATCH 046/309] Removed `_find_suitable_format` after its only use was deleted in the last commit. --- fs/_ftp_parse.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index ffbcbc65..9348ac5a 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -84,16 +84,6 @@ def parse_line(line): return None -def _find_suitable_format(t, formats): - for frmt in formats: - try: - if time.strptime(t, frmt): - return frmt - except ValueError: - continue - return None - - def _parse_time(t, formats): _t = None for frmt in formats: From d39b9852ab9f49c681db30fee257fe6d8bd3caa7 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 12 Dec 2020 08:49:08 +0100 Subject: [PATCH 047/309] Fixed a typo in a variable name in fs._ftp_parse by completely removing the reference in that line. --- fs/_ftp_parse.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 9348ac5a..5d892fdb 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -89,11 +89,10 @@ def _parse_time(t, formats): for frmt in formats: try: _t = time.strptime(t, frmt) - if t: - break + break except ValueError: continue - if not _t: + else: return None year = _t.tm_year if _t.tm_year != 1900 else time.localtime().tm_year From 1fa1f27e4e59ac69052cedc9bb7fece1fb2f57e1 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 12 Dec 2020 16:11:32 +0100 Subject: [PATCH 048/309] Removed redundant line in fs._ftp_parse. --- fs/_ftp_parse.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 5d892fdb..9e15a265 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -85,7 +85,6 @@ def parse_line(line): def _parse_time(t, formats): - _t = None for frmt in formats: try: _t = time.strptime(t, frmt) From 7e0f3b4fe5684b3a45555d70201360f1a6bdd3df Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 13:34:06 +0200 Subject: [PATCH 049/309] Update pull request template and move it to `.github` folder --- .../PULL_REQUEST_TEMPLATE.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) rename pull_request_template.md => .github/PULL_REQUEST_TEMPLATE.md (65%) diff --git a/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE.md similarity index 65% rename from pull_request_template.md rename to .github/PULL_REQUEST_TEMPLATE.md index 98b95212..6f7fcbcf 100644 --- a/pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,17 +1,19 @@ ## Type of changes -- [ ] Bug fix -- [ ] New feature -- [ ] Documentation / docstrings -- [ ] Tests -- [ ] Other + + +- Bug fix +- New feature +- Documentation / docstrings +- Tests +- Other ## Checklist - [ ] I've run the latest [black](https://github.com/ambv/black) with default args on new code. - [ ] I've updated CHANGELOG.md and CONTRIBUTORS.md where appropriate. - [ ] I've added tests for new code. -- [ ] I accept that @willmcgugan may be pedantic in the code review. +- [ ] I accept that @PyFilesystem/maintainers may be pedantic in the code review. ## Description From e3a9ee6d52e1551b1cdbc95beead982d3ce1d460 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 13:59:44 +0200 Subject: [PATCH 050/309] Move most of the metadata from `setup.py` to `setup.cfg` --- setup.cfg | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 47 ++--------------------------------------------- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/setup.cfg b/setup.cfg index a207101e..9a58e9ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,14 +2,63 @@ universal = 1 [metadata] +version = attr: fs._version.__version__ +name = fs +author = Will McGugan +author_email = will@willmcgugan.com +maintainer = Martin Larralde +maintainer_email = martin.larralde@embl.de +url = https://github.com/PyFilesystem/pyfilesystem2 +license = MIT license_file = LICENSE +description = Python's filesystem abstraction layer long_description = file: README.md long_description_content_type = text/markdown +platform = any +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: Implementation :: PyPy + Topic :: System :: Filesystems project_urls = Bug Reports = https://github.com/PyFilesystem/pyfilesystem2/issues Documentation = https://pyfilesystem2.readthedocs.io/en/latest/ Wiki = https://www.pyfilesystem.org/ +[options] +zip_safe = false +packages = find: +setup_requires = + setuptools >=38.3.0 +install_requires = + appdirs~=1.4.3 + pytz + setuptools + six ~=1.10 + enum34 ~=1.1.6 ; python_version < '3.4' + typing ~=3.6 ; python_version < '3.6' + backports.os ~=0.1 ; python_version < '3.0' + +[options.extras_require] +scandir = + scandir~=1.5 ; python_version < '3.5' + +[options.packages.find] +exclude = tests + +[options.package_data] +fs = py.typed + [pydocstyle] inherit = false ignore = D102,D105,D200,D203,D213,D406,D407 diff --git a/setup.py b/setup.py index a6ce6f43..ac42fe2c 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,4 @@ #!/usr/bin/env python -from setuptools import setup, find_packages - -with open("fs/_version.py") as f: - exec(f.read()) - -CLASSIFIERS = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: System :: Filesystems", -] - -REQUIREMENTS = ["appdirs~=1.4.3", "pytz", "setuptools", "six~=1.10"] - -setup( - author="Will McGugan", - author_email="will@willmcgugan.com", - classifiers=CLASSIFIERS, - description="Python's filesystem abstraction layer", - install_requires=REQUIREMENTS, - extras_require={ - "scandir :python_version < '3.5'": ["scandir~=1.5"], - ":python_version < '3.4'": ["enum34~=1.1.6"], - ":python_version < '3.6'": ["typing~=3.6"], - ":python_version < '3.0'": ["backports.os~=0.1"], - }, - license="MIT", - name="fs", - packages=find_packages(exclude=("tests",)), - package_data={"fs": ["py.typed"]}, - zip_safe=False, - platforms=["any"], - url="https://github.com/PyFilesystem/pyfilesystem2", - version=__version__, -) +from setuptools import setup +setup() From 20536d2b2579e5e53d626ed0f272c63dc1350af8 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 15:19:21 +0200 Subject: [PATCH 051/309] Update copyright line in `LICENSE` file --- LICENSE | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE b/LICENSE index 97a4b916..6f1b27cc 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,6 @@ MIT License +Copyright (c) 2017-2020 The PyFilesystem2 contributors Copyright (c) 2016-2019 Will McGugan Permission is hereby granted, free of charge, to any person obtaining a copy From 589d9a8251c91d0272ad5c881864d27e4baf7565 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 15:22:49 +0200 Subject: [PATCH 052/309] Bump version to `v2.4.12` --- CHANGELOG.md | 2 +- fs/_version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e0a7680..c50b777b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.12] - (Unreleased) +## [2.4.12] - 2020-10-18 ### Added diff --git a/fs/_version.py b/fs/_version.py index 0e4134f2..7dc85736 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.11" +__version__ = "2.4.12" From e4a3f7101b1720474259cc28de6127671b3ad664 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 15:27:17 +0200 Subject: [PATCH 053/309] Update names in `README.md` and `CONTRIBUTORS.md` --- CONTRIBUTORS.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 5df27157..619e1c13 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,7 +6,7 @@ Many thanks to the following developers for contributing to this project: - [C. W.](https://github.com/chfw) - [Diego Argueta](https://github.com/dargueta) - [Geoff Jukes](https://github.com/geoffjukes) -- [Giampaolo](https://github.com/gpcimino) +- [Giampaolo Cimino](https://github.com/gpcimino) - [Justin Charlong](https://github.com/jcharlong) - [Louis Sautier](https://github.com/sbraz) - [Martin Larralde](https://github.com/althonos) diff --git a/README.md b/README.md index fbd6b200..787b29ec 100644 --- a/README.md +++ b/README.md @@ -95,10 +95,11 @@ The following developers have contributed code and their time to this projects: - [Will McGugan](https://github.com/willmcgugan) - [Martin Larralde](https://github.com/althonos) -- [Giampaolo](https://github.com/gpcimino) +- [Giampaolo Cimino](https://github.com/gpcimino) - [Geoff Jukes](https://github.com/geoffjukes) -See CONTRIBUTORS.md for a full list of contributors. +See [CONTRIBUTORS.md](https://github.com/PyFilesystem/pyfilesystem2/blob/master/CONTRIBUTORS.md) +for a full list of contributors. PyFilesystem2 owes a massive debt of gratitude to the following developers who contributed code and ideas to the original version. From 6f234fac9d230c5490013279a1dee8b65b4b1a10 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 15:28:55 +0200 Subject: [PATCH 054/309] Make Travis-CI build against Python 3.9 development version --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 72437a67..21a020fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9-dev" - "pypy" - "pypy3.5-7.0" # Need 7.0+ due to a bug in earlier versions that broke our tests. From 480bc6ed61db67eee59d8668eab700444d08c276 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 18 Oct 2020 15:48:30 +0200 Subject: [PATCH 055/309] Fix `setup.py` to work with Python 2.7 --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac42fe2c..756c6bef 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,9 @@ #!/usr/bin/env python +import os + +with open(os.path.join("fs", "_version.py")) as f: + exec(f.read()) + from setuptools import setup -setup() +setup(version=__version__) From c17b4680f616000bcc27aa43d1ceab6c1869f773 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 13 Jan 2021 15:00:04 +0100 Subject: [PATCH 056/309] Add Python 3.9 to the supported versions in `setup.cfg` --- CHANGELOG.md | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c50b777b..b16e84a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.12] - 2020-10-18 +## [2.4.12] - 2021-01-13 ### Added diff --git a/setup.cfg b/setup.cfg index 9a58e9ee..24c9f3d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ classifiers = Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: System :: Filesystems From f4fa74de879f1b2e077086e6d97e874c24f22cf6 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 13 Jan 2021 15:02:04 +0100 Subject: [PATCH 057/309] Bump year in `LICENSE` file --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6f1b27cc..34845692 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017-2020 The PyFilesystem2 contributors +Copyright (c) 2017-2021 The PyFilesystem2 contributors Copyright (c) 2016-2019 Will McGugan Permission is hereby granted, free of charge, to any person obtaining a copy From a55c9ff248fae36cddb1004a3f9dcbb631b3a759 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 14 Jan 2021 14:55:07 +0100 Subject: [PATCH 058/309] Add a CI stage deploying tagged commits to PyPI using `twine` --- .travis.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 21a020fa..89b1cce9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ python: - "3.6" - "3.7" - "3.8" - - "3.9-dev" + - "3.9" - "pypy" - "pypy3.5-7.0" # Need 7.0+ due to a bug in earlier versions that broke our tests. @@ -36,8 +36,22 @@ before_install: install: - pip install -e . +# command to run tests +script: tox + after_success: - coveralls -# command to run tests -script: tox +before_deploy: + - pip install -U twine wheel + - python setup.py sdist bdist_wheel + +deploy: + provider: script + script: twine upload dist/* + skip_cleanup: true + on: + python: 3.9 + tags: true + repo: PyFilesystem/pyfilesystem2 + From 9bd26ac1758f988961e07a11b893b9254980e204 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 14 Jan 2021 15:13:26 +0100 Subject: [PATCH 059/309] Release v2.4.12 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b16e84a9..9e9a5c00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). -## [2.4.12] - 2021-01-13 +## [2.4.12] - 2021-01-14 ### Added From 22f19e3d611fb4c073911d73f9d26ca902f3a422 Mon Sep 17 00:00:00 2001 From: Eelke van den Bos Date: Wed, 20 Jan 2021 16:58:14 +0100 Subject: [PATCH 060/309] Update base.py --- fs/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/base.py b/fs/base.py index 5eb0b90b..a6c52bcb 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1308,6 +1308,10 @@ def upload(self, path, file, chunk_size=None, **options): sensible default. **options: Implementation specific options required to open the source file. + + Raises: + fs.errors.ResourceNotFound: If a parent directory of + ``path`` does not exist. Note that the file object ``file`` will *not* be closed by this method. Take care to close it after this method completes From 74fe141b73c495afeb059b92903f8a7f9227c0d5 Mon Sep 17 00:00:00 2001 From: Eelke van den Bos Date: Tue, 26 Jan 2021 16:58:43 +0100 Subject: [PATCH 061/309] Assert fs.upload raises ResourceNotFound --- fs/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/test.py b/fs/test.py index 9d8e7ed7..8b8cb9fd 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1494,6 +1494,10 @@ def test_upload(self): with self.fs.open("foo", "rb") as f: data = f.read() self.assertEqual(data, b"bar") + + # upload to non-existing path (/foo/bar) + with self.assertRaises(errors.ResourceNotFound): + self.fs.upload("/foo/bar/baz", bytes_file) def test_upload_chunk_size(self): test_data = b"bar" * 128 From ad97c745f05e32166995fd04219996a86d80d95a Mon Sep 17 00:00:00 2001 From: Eelke van den Bos Date: Tue, 26 Jan 2021 20:58:01 +0100 Subject: [PATCH 062/309] Improve ResourceNotFound assertion in test_upload --- fs/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/test.py b/fs/test.py index 8b8cb9fd..de70f280 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1495,9 +1495,9 @@ def test_upload(self): data = f.read() self.assertEqual(data, b"bar") - # upload to non-existing path (/foo/bar) + # upload to non-existing path (/spam/eggs) with self.assertRaises(errors.ResourceNotFound): - self.fs.upload("/foo/bar/baz", bytes_file) + self.fs.upload("/spam/eggs", bytes_file) def test_upload_chunk_size(self): test_data = b"bar" * 128 From 6b30f6a7cf3be81a403599a0ef16c31c16319698 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 27 Jan 2021 00:47:11 +0100 Subject: [PATCH 063/309] Remove line with whitespaces in `fs.base` failing `flake8` linter --- fs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index a6c52bcb..286cd025 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1308,7 +1308,7 @@ def upload(self, path, file, chunk_size=None, **options): sensible default. **options: Implementation specific options required to open the source file. - + Raises: fs.errors.ResourceNotFound: If a parent directory of ``path`` does not exist. From 89ef78f54127eb88bb36a328429f79cd8f066bf3 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 27 Jan 2021 01:48:51 +0100 Subject: [PATCH 064/309] Update `CHANGELOG.md` and `CONTRIBUTORS.md` with @eelkevdbos patch --- CHANGELOG.md | 8 ++++++++ CONTRIBUTORS.md | 1 + 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e9a5c00..48676340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). + +## Unreleased + +### Changed +- Make `FS.upload` explicit about the expected error when the parent directory of the destination does not exist + [#445](https://github.com/PyFilesystem/pyfilesystem2/pull/445). + + ## [2.4.12] - 2021-01-14 ### Added diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 619e1c13..3c2729c7 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -5,6 +5,7 @@ Many thanks to the following developers for contributing to this project: - [Andreas Tollkötter](https://github.com/atollk) - [C. W.](https://github.com/chfw) - [Diego Argueta](https://github.com/dargueta) +- [Eelke van den Bos](https://github.com/eelkevdbos) - [Geoff Jukes](https://github.com/geoffjukes) - [Giampaolo Cimino](https://github.com/gpcimino) - [Justin Charlong](https://github.com/jcharlong) From 5dedf615b20cabe0dd8440545fee3b3d110ce68f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 00:41:06 +0100 Subject: [PATCH 065/309] Attempt setup of GitHub Actions to run tests with `tox` --- .github/workflows/test.yml | 30 ++++++++++++++++++++++++++++++ tox.ini | 14 ++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..ad9896f5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: Test + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - 2.7 + - 3.4 + - 3.5 + - 3.6 + - 3.7 + - 3.8 + - 3.9 + steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Installing tox + run: python -m pip install -U pip tox + - name: Test with tox + run: tox diff --git a/tox.ini b/tox.ini index 887ecd56..f6d00e75 100644 --- a/tox.ini +++ b/tox.ini @@ -8,9 +8,9 @@ deps = -r {toxinidir}/testrequirements.txt commands = coverage run -m pytest --cov-append {posargs} {toxinidir}/tests [testenv:typecheck] -python = python37 +python = python39 deps = - mypy==0.740 + mypy==0.800 -r {toxinidir}/testrequirements.txt commands = make typecheck whitelist_externals = make @@ -25,3 +25,13 @@ deps = # flake8-isort flake8-mutable commands = flake8 fs tests + +[gh-actions] +python = + 2.7: py27, py27-scandir + 3.4: py34, py34-scandir + 3.6: py35, py35-scandir + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39, typecheck, lint From 8c0b77148fbbaf4c13edb34c61e92a34e584e35c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 00:51:56 +0100 Subject: [PATCH 066/309] Install `tox-gh-actions` in GitHub Actions test workflow --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad9896f5..55864713 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Installing tox - run: python -m pip install -U pip tox + - name: Install tox + run: python -m pip install -U pip tox tox-gh-actions - name: Test with tox run: tox From a2ebaaef3f88cc915a42f99b04808802fddfdff4 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 00:54:51 +0100 Subject: [PATCH 067/309] Make sure to use a recent version of `pip` in GitHub Actions --- .github/workflows/test.yml | 7 ++++--- tox.ini | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55864713..7caa82c2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,6 @@ jobs: matrix: python-version: - 2.7 - - 3.4 - 3.5 - 3.6 - 3.7 @@ -24,7 +23,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Update pip + run: python -m pip install -U pip wheel - name: Install tox - run: python -m pip install -U pip tox tox-gh-actions + run: python -m pip install tox tox-gh-actions - name: Test with tox - run: tox + run: python -m tox diff --git a/tox.ini b/tox.ini index f6d00e75..e165ddef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = {py27,py34,py35,py36,py37}{,-scandir},pypy,typecheck,lint +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, typecheck, lint sitepackages = False skip_missing_interpreters=True @@ -30,7 +30,7 @@ commands = flake8 fs tests python = 2.7: py27, py27-scandir 3.4: py34, py34-scandir - 3.6: py35, py35-scandir + 3.5: py35, py35-scandir 3.6: py36 3.7: py37 3.8: py38 From 1c71b8e790b433e643e5fc16eea1d6b8d78356cf Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 01:25:01 +0100 Subject: [PATCH 068/309] Make CI jobs upload coverage statistics to Coveralls --- .github/workflows/test.yml | 18 ++++++++++++++++-- setup.cfg | 1 + tox.ini | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7caa82c2..7a014702 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: - pull_request jobs: - build: + test: runs-on: ubuntu-latest strategy: matrix: @@ -24,8 +24,22 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install -U pip wheel + run: python -m pip install -U pip wheel setuptools - name: Install tox run: python -m pip install tox tox-gh-actions - name: Test with tox run: python -m tox + - name: Collect coverage results + uses: AndreMiras/coveralls-python-action@develop + with: + parallel: true + flag-name: test (${{ matrix.python-version}}) + + coveralls: + needs: test + runs-on: ubuntu-latest + steps: + - name: Upload coverage statistics to Coveralls + uses: AndreMiras/coveralls-python-action@develop + with: + parallel-finished: true diff --git a/setup.cfg b/setup.cfg index 24c9f3d7..109173e0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,7 @@ disallow_untyped_defs = false branch = true omit = fs/test.py source = fs +relative_files = true [coverage:report] show_missing = true diff --git a/tox.ini b/tox.ini index e165ddef..1fa6a035 100644 --- a/tox.ini +++ b/tox.ini @@ -34,4 +34,4 @@ python = 3.6: py36 3.7: py37 3.8: py38 - 3.9: py39, typecheck, lint + 3.9: py39 From 9e726ce52cfc42eebe5aa121492863be26167492 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 02:54:00 +0100 Subject: [PATCH 069/309] Make minimal `requirements.txt` files for the docs and the tests --- docs/requirements.txt | 3 +++ tests/requirements.txt | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 docs/requirements.txt create mode 100644 tests/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..c2d9a973 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +# the bare requirements for building docs +Sphinx ~=3.0 +sphinx-rtd-theme ~=0.5.1 diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..269c6039 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,4 @@ +# the bare requirements for running tests +pyftpdlib ~=1.5 +psutil ~=5.0 +pysendfile ~=2.0 ; python_version <= "3.3" From c824bdbc9359ebd5b85bc23703f8fd7a6f1f560b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 02:57:01 +0100 Subject: [PATCH 070/309] Refactor `tox.ini` to pack its own dependencies --- tox.ini | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/tox.ini b/tox.ini index 1fa6a035..ce31fdb4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,36 +1,47 @@ [tox] -envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, typecheck, lint -sitepackages = False -skip_missing_interpreters=True +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, mypy, lint +sitepackages = false +skip_missing_interpreters = true [testenv] -deps = -r {toxinidir}/testrequirements.txt commands = coverage run -m pytest --cov-append {posargs} {toxinidir}/tests +deps = + -rtests/requirements.txt + pytest-cov~=2.11 + py{36,37,38,39}: pytest~=6.2 + py{27,34}: pytest~=4.6 + py{36,37,38,39}: pytest-randomly~=3.5 + py{27,34}: pytest-randomly~=1.2 + scandir: .[scandir] + !scandir: . -[testenv:typecheck] -python = python39 +[testenv:mypy] +basepython = python3.9 +commands = mypy --config-file {toxinidir}/setup.cfg {toxinidir}/fs deps = + . mypy==0.800 - -r {toxinidir}/testrequirements.txt -commands = make typecheck -whitelist_externals = make [testenv:lint] -python = python37 +python = python3.9 +commands = flake8 fs tests deps = flake8 - # flake8-builtins flake8-bugbear flake8-comprehensions - # flake8-isort flake8-mutable -commands = flake8 fs tests + +[testenv:format] +python = python3.9 +commands = black +deps = + black [gh-actions] python = 2.7: py27, py27-scandir 3.4: py34, py34-scandir - 3.5: py35, py35-scandir + 3.5: py35 3.6: py36 3.7: py37 3.8: py38 From ad127ffde22f8cbda4afc094ba387ae30ee15c30 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 03:37:39 +0100 Subject: [PATCH 071/309] Rewrite tests to use `unittest.skipIf` instead of `pytest.mark.skipif` decorator --- tests/requirements.txt | 7 +++++++ tests/test_encoding.py | 7 +++---- tests/test_memoryfs.py | 7 +++---- tests/test_osfs.py | 9 ++++----- tests/test_tarfs.py | 3 +-- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 269c6039..d5b9a2f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,11 @@ # the bare requirements for running tests + +# pyftpdlib is needed to spawn a FTP server for the +# FTPFS test suite pyftpdlib ~=1.5 + +# these are optional dependencies for pyftpdlib that +# are not explicitly listed, we need to install these +# ourselves psutil ~=5.0 pysendfile ~=2.0 ; python_version <= "3.3" diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 59659942..c8782ff2 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -6,8 +6,6 @@ import tempfile import unittest -import pytest - import six import fs @@ -16,8 +14,9 @@ if platform.system() != "Windows": - @pytest.mark.skipif( - platform.system() == "Darwin", reason="Bad unicode not possible on OSX" + @unittest.skipIf( + platform.system() == "Darwin", + "Bad unicode not possible on OSX" ) class TestEncoding(unittest.TestCase): diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index c8193fd6..842c41fb 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -3,8 +3,6 @@ import posixpath import unittest -import pytest - from fs import memoryfs from fs.test import FSTestCases from fs.test import UNICODE_TEXT @@ -30,8 +28,9 @@ def _create_many_files(self): posixpath.join(parent_dir, str(file_id)), UNICODE_TEXT ) - @pytest.mark.skipif( - not tracemalloc, reason="`tracemalloc` isn't supported on this Python version." + @unittest.skipUnless( + tracemalloc, + reason="`tracemalloc` isn't supported on this Python version." ) def test_close_mem_free(self): """Ensure all file memory is freed when calling close(). diff --git a/tests/test_osfs.py b/tests/test_osfs.py index f656646c..e43635f4 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -8,7 +8,6 @@ import tempfile import sys import unittest -import pytest from fs import osfs, open_fs from fs.path import relpath, dirname @@ -88,10 +87,10 @@ def test_expand_vars(self): self.assertIn("TYRIONLANISTER", fs1.getsyspath("/")) self.assertNotIn("TYRIONLANISTER", fs2.getsyspath("/")) - @pytest.mark.skipif(osfs.sendfile is None, reason="sendfile not supported") - @pytest.mark.skipif( + @unittest.skipUnless(osfs.sendfile, "sendfile not supported") + @unittest.skipIf( sys.version_info >= (3, 8), - reason="the copy function uses sendfile in Python 3.8+, " + "the copy function uses sendfile in Python 3.8+, " "making the patched implementation irrelevant", ) def test_copy_sendfile(self): @@ -139,7 +138,7 @@ def test_unicode_paths(self): finally: shutil.rmtree(dir_path) - @pytest.mark.skipif(not hasattr(os, "symlink"), reason="No symlink support") + @unittest.skipUnless(hasattr(os, "symlink"), "No symlink support") def test_symlinks(self): with open(self._get_real_path("foo"), "wb") as f: f.write(b"foobar") diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index a90dc0ea..c518e8c4 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -7,7 +7,6 @@ import tarfile import tempfile import unittest -import pytest from fs import tarfs from fs.enums import ResourceType @@ -94,7 +93,7 @@ def destroy_fs(self, fs): del fs._tar_file -@pytest.mark.skipif(six.PY2, reason="Python2 does not support LZMA") +@unittest.skipIf(six.PY2, "Python2 does not support LZMA") class TestWriteXZippedTarFS(FSTestCases, unittest.TestCase): def make_fs(self): fh, _tar_file = tempfile.mkstemp() From 2c19154c3ff0ac70bdc54af30c37db9fbdc0ea01 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 03:46:21 +0100 Subject: [PATCH 072/309] Rewrite `tests.test_appfs` as a proper unittest module --- tests/test_appfs.py | 82 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/tests/test_appfs.py b/tests/test_appfs.py index a060e97a..d673c9b2 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -1,24 +1,80 @@ from __future__ import unicode_literals -import pytest +import os +import shutil +import tempfile +import unittest + import six +try: + from unittest import mock +except ImportError: + import mock + +import fs.test from fs import appfs -@pytest.fixture -def fs(mock_appdir_directories): - """Create a UserDataFS but strictly using a temporary directory.""" - return appfs.UserDataFS("fstest", "willmcgugan", "1.0") +class _TestAppFS(object): + + AppFS = None + + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.mkdtemp() + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmpdir) + + def make_fs(self): + with mock.patch( + "appdirs.{}".format(self.AppFS.app_dir), + autospec=True, + spec_set=True, + return_value=tempfile.mkdtemp(dir=self.tmpdir) + ): + return self.AppFS("fstest", "willmcgugan", "1.0") + + if six.PY2: + + def test_repr(self): + self.assertEqual( + repr(self.fs), + "{}(u'fstest', author=u'willmcgugan', version=u'1.0')".format(self.AppFS.__name__) + ) + + else: + + def test_repr(self): + self.assertEqual( + repr(self.fs), + "{}('fstest', author='willmcgugan', version='1.0')".format(self.AppFS.__name__) + ) + + + def test_str(self): + self.assertEqual( + str(self.fs), + "<{} 'fstest'>".format(self.AppFS.__name__.lower()) + ) + + +class TestUserDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.UserDataFS + +class TestUserConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.UserConfigFS +class TestUserCacheFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.UserCacheFS -@pytest.mark.skipif(six.PY2, reason="Test requires Python 3 repr") -def test_user_data_repr_py3(fs): - assert repr(fs) == "UserDataFS('fstest', author='willmcgugan', version='1.0')" - assert str(fs) == "" +class TestSiteDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.SiteDataFS +class TestSiteConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.SiteConfigFS -@pytest.mark.skipif(not six.PY2, reason="Test requires Python 2 repr") -def test_user_data_repr_py2(fs): - assert repr(fs) == "UserDataFS(u'fstest', author=u'willmcgugan', version=u'1.0')" - assert str(fs) == "" +class TestUserLogFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): + AppFS = appfs.UserLogFS From b2f40ec51938700c85200e50f2b24d12ba7f8ef8 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 03:47:00 +0100 Subject: [PATCH 073/309] Make `tests.test_ftpfs` use `pytest.mark.slow` only if available --- tests/test_ftpfs.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 7207e9e9..530e020e 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -12,7 +12,6 @@ import unittest import uuid -import pytest from six import text_type from ftplib import error_perm @@ -27,6 +26,13 @@ from fs.subfs import SubFS from fs.test import FSTestCases +try: + from pytest.mark import slow +except: + def slow(cls): + return cls + + # Prevent socket timeouts from slowing tests too much socket.setdefaulttimeout(1) @@ -129,7 +135,7 @@ def test_manager_with_host(self): ) -@pytest.mark.slow +@slow class TestFTPFS(FSTestCases, unittest.TestCase): user = "user" @@ -279,7 +285,7 @@ def test_features(self): pass -@pytest.mark.slow +@slow class TestAnonFTPFS(FSTestCases, unittest.TestCase): user = "anonymous" From 188d0a8d3682013d215136cfc0df8b64f14d1daa Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 03:50:47 +0100 Subject: [PATCH 074/309] Rewrite `tests.test_opener` without using `pytest` fixtures --- tests/conftest.py | 34 ---------------------------------- tests/test_opener.py | 26 ++++++++++++++++++++------ 2 files changed, 20 insertions(+), 40 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index b820712f..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest - -try: - from unittest import mock -except ImportError: - import mock - - -@pytest.fixture -@mock.patch("appdirs.user_data_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.site_data_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.user_config_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.site_config_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.user_cache_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.user_state_dir", autospec=True, spec_set=True) -@mock.patch("appdirs.user_log_dir", autospec=True, spec_set=True) -def mock_appdir_directories( - user_log_dir_mock, - user_state_dir_mock, - user_cache_dir_mock, - site_config_dir_mock, - user_config_dir_mock, - site_data_dir_mock, - user_data_dir_mock, - tmpdir -): - """Mock out every single AppDir directory so tests can't access real ones.""" - user_log_dir_mock.return_value = str(tmpdir.join("user_log").mkdir()) - user_state_dir_mock.return_value = str(tmpdir.join("user_state").mkdir()) - user_cache_dir_mock.return_value = str(tmpdir.join("user_cache").mkdir()) - site_config_dir_mock.return_value = str(tmpdir.join("site_config").mkdir()) - user_config_dir_mock.return_value = str(tmpdir.join("user_config").mkdir()) - site_data_dir_mock.return_value = str(tmpdir.join("site_data").mkdir()) - user_data_dir_mock.return_value = str(tmpdir.join("user_data").mkdir()) diff --git a/tests/test_opener.py b/tests/test_opener.py index fc450751..b01e8d4e 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -1,13 +1,12 @@ from __future__ import unicode_literals import os +import shutil import sys import tempfile import unittest import pkg_resources -import pytest - from fs import open_fs, opener from fs.osfs import OSFS from fs.opener import registry, errors @@ -208,8 +207,14 @@ def test_manage_fs_error(self): self.assertTrue(mem_fs.isclosed()) -@pytest.mark.usefixtures("mock_appdir_directories") class TestOpeners(unittest.TestCase): + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdir) + def test_repr(self): # Check __repr__ works for entry_point in pkg_resources.iter_entry_points("fs.opener"): @@ -260,7 +265,10 @@ def test_open_fs(self): mem_fs_2 = opener.open_fs(mem_fs) self.assertEqual(mem_fs, mem_fs_2) - def test_open_userdata(self): + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + def test_open_userdata(self, app_dir): + app_dir.return_value = self.tmpdir + with self.assertRaises(errors.OpenerError): opener.open_fs("userdata://foo:bar:baz:egg") @@ -269,13 +277,19 @@ def test_open_userdata(self): self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, "1.0") - def test_open_userdata_no_version(self): + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + def test_open_userdata_no_version(self, app_dir): + app_dir.return_value = self.tmpdir + app_fs = opener.open_fs("userdata://fstest:willmcgugan", create=True) self.assertEqual(app_fs.app_dirs.appname, "fstest") self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, None) - def test_user_data_opener(self): + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + def test_user_data_opener(self, app_dir): + app_dir.return_value = self.tmpdir + user_data_fs = open_fs("userdata://fstest:willmcgugan:1.0", create=True) self.assertIsInstance(user_data_fs, UserDataFS) user_data_fs.makedir("foo", recreate=True) From 1914a6e1714a43e23bbdb938cd10dd1fd75231b0 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 03:56:32 +0100 Subject: [PATCH 075/309] Fix missing `mock` test requirement for Python 2.7 --- tests/requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/requirements.txt b/tests/requirements.txt index d5b9a2f5..9e7ece32 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,3 +9,8 @@ pyftpdlib ~=1.5 # ourselves psutil ~=5.0 pysendfile ~=2.0 ; python_version <= "3.3" + +# mock is only available from Python 3.3 onward, and +# mock v4+ doesn't support Python 2.7 anymore +mock ~=3.0 ; python_version < "3.3" + From aede86010c19dffccfe173b94f4557377328f80e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 04:01:31 +0100 Subject: [PATCH 076/309] Fix `tox` not correctly collecting coverage with `pytest-cov` --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ce31fdb4..361e69ff 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ sitepackages = false skip_missing_interpreters = true [testenv] -commands = coverage run -m pytest --cov-append {posargs} {toxinidir}/tests +commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests deps = -rtests/requirements.txt pytest-cov~=2.11 From 07f4bc61b29c3099b54e2cd9b0d6476ce59b8a99 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 04:07:11 +0100 Subject: [PATCH 077/309] Require minimum `coverage` version of `5.0` to use the `relative_files` option --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 361e69ff..18e9b353 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests deps = -rtests/requirements.txt pytest-cov~=2.11 + coverage~=5.0 py{36,37,38,39}: pytest~=6.2 py{27,34}: pytest~=4.6 py{36,37,38,39}: pytest-randomly~=3.5 From c23957403995a2fcff7ce000bba102b4af4d756e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 04:21:50 +0100 Subject: [PATCH 078/309] Add a linting stage to the GitHub Actions test workflow --- .github/workflows/test.yml | 24 ++++++++++++++++++++++++ tox.ini | 13 +++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7a014702..c9647b2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,3 +43,27 @@ jobs: uses: AndreMiras/coveralls-python-action@develop with: parallel-finished: true + + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + linter: + - mypy + - flake8 + - black + - pydocstyle + steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Setup Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Update pip + run: python -m pip install -U pip wheel setuptools + - name: Install tox + run: python -m pip install tox tox-gh-actions + - name: Run ${{ matrix.linter }} linter + run: python -m tox -e ${{ matrix.linter }} diff --git a/tox.ini b/tox.ini index 18e9b353..d1b6739e 100644 --- a/tox.ini +++ b/tox.ini @@ -23,21 +23,26 @@ deps = . mypy==0.800 -[testenv:lint] +[testenv:flake8] python = python3.9 -commands = flake8 fs tests +commands = flake8 {toxinidir}/fs {toxinidir}/tests deps = flake8 flake8-bugbear + flake8-builtins flake8-comprehensions flake8-mutable -[testenv:format] +[testenv:black] python = python3.9 -commands = black +commands = black --check {toxinidir}/fs deps = black +[testenv:pydocstyle] +python = python3.9 +commands = pydocstyle --config={toxinidir}/setup.cfg {toxinidir}/fs + [gh-actions] python = 2.7: py27, py27-scandir From 87830912b0f37adda38b77c97db576a7bbe2b313 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 04:39:06 +0100 Subject: [PATCH 079/309] Add PyPy environments to `tox.ini` configuration --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d1b6739e..ec1d7037 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, mypy, lint +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, mypy, lint sitepackages = false skip_missing_interpreters = true From eaae371b452d00798866cfe86d86c920c2f8e9cd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 04:54:39 +0100 Subject: [PATCH 080/309] Reformat Python code with latest version of `black` --- fs/appfs.py | 3 +- fs/base.py | 33 ++++++----------- fs/error_tools.py | 3 +- fs/errors.py | 84 ++++++++++++++---------------------------- fs/ftpfs.py | 18 +++------ fs/glob.py | 20 +++++----- fs/info.py | 18 +++------ fs/iotools.py | 6 +-- fs/lrucache.py | 6 +-- fs/memoryfs.py | 12 ++---- fs/mode.py | 30 +++++---------- fs/mountfs.py | 3 +- fs/multifs.py | 15 +++----- fs/opener/appfs.py | 3 +- fs/opener/errors.py | 15 +++----- fs/opener/ftpfs.py | 3 +- fs/opener/memoryfs.py | 3 +- fs/opener/osfs.py | 3 +- fs/opener/registry.py | 7 ++-- fs/opener/tarfs.py | 3 +- fs/opener/tempfs.py | 3 +- fs/opener/zipfs.py | 3 +- fs/osfs.py | 17 +++------ fs/permissions.py | 27 +++++--------- fs/subfs.py | 3 +- fs/tarfs.py | 6 +-- fs/tempfs.py | 3 +- fs/test.py | 15 +++----- fs/time.py | 6 +-- fs/tree.py | 24 ++++-------- fs/walk.py | 12 ++---- fs/wildcard.py | 4 +- fs/zipfs.py | 12 ++---- tests/test_appfs.py | 19 +++++++--- tests/test_encoding.py | 5 +-- tests/test_errors.py | 2 +- tests/test_ftpfs.py | 2 +- tests/test_info.py | 1 - tests/test_memoryfs.py | 3 +- tests/test_move.py | 1 - tests/test_opener.py | 1 - tests/test_tarfs.py | 3 +- tox.ini | 2 +- 43 files changed, 165 insertions(+), 297 deletions(-) diff --git a/fs/appfs.py b/fs/appfs.py index dafe2e98..47223e83 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -30,8 +30,7 @@ class _AppFS(OSFS): - """Abstract base class for an app FS. - """ + """Abstract base class for an app FS.""" # FIXME(@althonos): replace by ClassVar[Text] once # https://github.com/python/mypy/pull/4718 is accepted diff --git a/fs/base.py b/fs/base.py index 286cd025..7271a7ca 100644 --- a/fs/base.py +++ b/fs/base.py @@ -92,8 +92,7 @@ def _method(*args, **kwargs): @six.add_metaclass(abc.ABCMeta) class FS(object): - """Base class for FS objects. - """ + """Base class for FS objects.""" # This is the "standard" meta namespace. _meta = {} # type: Dict[Text, Union[Text, int, bool, None]] @@ -106,8 +105,7 @@ class FS(object): def __init__(self): # type: (...) -> None - """Create a filesystem. See help(type(self)) for accurate signature. - """ + """Create a filesystem. See help(type(self)) for accurate signature.""" self._closed = False self._lock = threading.RLock() super(FS, self).__init__() @@ -118,8 +116,7 @@ def __del__(self): def __enter__(self): # type: (...) -> FS - """Allow use of filesystem as a context manager. - """ + """Allow use of filesystem as a context manager.""" return self def __exit__( @@ -129,21 +126,18 @@ def __exit__( traceback, # type: Optional[TracebackType] ): # type: (...) -> None - """Close filesystem on exit. - """ + """Close filesystem on exit.""" self.close() @property def glob(self): - """`~fs.glob.BoundGlobber`: a globber object.. - """ + """`~fs.glob.BoundGlobber`: a globber object..""" return BoundGlobber(self) @property def walk(self): # type: (_F) -> BoundWalker[_F] - """`~fs.walk.BoundWalker`: a walker bound to this filesystem. - """ + """`~fs.walk.BoundWalker`: a walker bound to this filesystem.""" return self.walker_class.bind(self) # ---------------------------------------------------------------- # @@ -544,26 +538,22 @@ def filterdir( def match_dir(patterns, info): # type: (Optional[Iterable[Text]], Info) -> bool - """Pattern match info.name. - """ + """Pattern match info.name.""" return info.is_file or self.match(patterns, info.name) def match_file(patterns, info): # type: (Optional[Iterable[Text]], Info) -> bool - """Pattern match info.name. - """ + """Pattern match info.name.""" return info.is_dir or self.match(patterns, info.name) def exclude_dir(patterns, info): # type: (Optional[Iterable[Text]], Info) -> bool - """Pattern match info.name. - """ + """Pattern match info.name.""" return info.is_file or not self.match(patterns, info.name) def exclude_file(patterns, info): # type: (Optional[Iterable[Text]], Info) -> bool - """Pattern match info.name. - """ + """Pattern match info.name.""" return info.is_dir or not self.match(patterns, info.name) if files: @@ -914,8 +904,7 @@ def hasurl(self, path, purpose="download"): def isclosed(self): # type: () -> bool - """Check if the filesystem is closed. - """ + """Check if the filesystem is closed.""" return getattr(self, "_closed", False) def isdir(self, path): diff --git a/fs/error_tools.py b/fs/error_tools.py index 28c200bf..66d38696 100644 --- a/fs/error_tools.py +++ b/fs/error_tools.py @@ -28,8 +28,7 @@ class _ConvertOSErrors(object): - """Context manager to convert OSErrors in to FS Errors. - """ + """Context manager to convert OSErrors in to FS Errors.""" FILE_ERRORS = { 64: errors.RemoteConnectionError, # ENONET diff --git a/fs/errors.py b/fs/errors.py index b70b62e3..49bc2d6f 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -55,8 +55,7 @@ class MissingInfoNamespace(AttributeError): - """An expected namespace is missing. - """ + """An expected namespace is missing.""" def __init__(self, namespace): # type: (Text) -> None @@ -70,8 +69,7 @@ def __reduce__(self): @six.python_2_unicode_compatible class FSError(Exception): - """Base exception for the `fs` module. - """ + """Base exception for the `fs` module.""" default_message = "Unspecified error" @@ -82,8 +80,7 @@ def __init__(self, msg=None): def __str__(self): # type: () -> Text - """Return the error message. - """ + """Return the error message.""" msg = self._msg.format(**self.__dict__) return msg @@ -94,8 +91,7 @@ def __repr__(self): class FilesystemClosed(FSError): - """Attempt to use a closed filesystem. - """ + """Attempt to use a closed filesystem.""" default_message = "attempt to use closed filesystem" @@ -111,8 +107,7 @@ def __init__(self, errors): class CreateFailed(FSError): - """Filesystem could not be created. - """ + """Filesystem could not be created.""" default_message = "unable to create filesystem, {details}" @@ -140,8 +135,7 @@ def __reduce__(self): class PathError(FSError): - """Base exception for errors to do with a path string. - """ + """Base exception for errors to do with a path string.""" default_message = "path '{path}' is invalid" @@ -155,15 +149,13 @@ def __reduce__(self): class NoSysPath(PathError): - """The filesystem does not provide *sys paths* to the resource. - """ + """The filesystem does not provide *sys paths* to the resource.""" default_message = "path '{path}' does not map to the local filesystem" class NoURL(PathError): - """The filesystem does not provide an URL for the resource. - """ + """The filesystem does not provide an URL for the resource.""" default_message = "path '{path}' has no '{purpose}' URL" @@ -177,22 +169,19 @@ def __reduce__(self): class InvalidPath(PathError): - """Path can't be mapped on to the underlaying filesystem. - """ + """Path can't be mapped on to the underlaying filesystem.""" default_message = "path '{path}' is invalid on this filesystem " class InvalidCharsInPath(InvalidPath): - """Path contains characters that are invalid on this filesystem. - """ + """Path contains characters that are invalid on this filesystem.""" default_message = "path '{path}' contains invalid characters" class OperationFailed(FSError): - """A specific operation failed. - """ + """A specific operation failed.""" default_message = "operation failed, {details}" @@ -214,50 +203,43 @@ def __reduce__(self): class Unsupported(OperationFailed): - """Operation not supported by the filesystem. - """ + """Operation not supported by the filesystem.""" default_message = "not supported" class RemoteConnectionError(OperationFailed): - """Operations encountered remote connection trouble. - """ + """Operations encountered remote connection trouble.""" default_message = "remote connection error" class InsufficientStorage(OperationFailed): - """Storage is insufficient for requested operation. - """ + """Storage is insufficient for requested operation.""" default_message = "insufficient storage space" class PermissionDenied(OperationFailed): - """Not enough permissions. - """ + """Not enough permissions.""" default_message = "permission denied" class OperationTimeout(OperationFailed): - """Filesystem took too long. - """ + """Filesystem took too long.""" default_message = "operation timed out" class RemoveRootError(OperationFailed): - """Attempt to remove the root directory. - """ + """Attempt to remove the root directory.""" default_message = "root directory may not be removed" class ResourceError(FSError): - """Base exception class for error associated with a specific resource. - """ + """Base exception class for error associated with a specific resource.""" default_message = "failed on path {path}" @@ -272,71 +254,61 @@ def __reduce__(self): class ResourceNotFound(ResourceError): - """Required resource not found. - """ + """Required resource not found.""" default_message = "resource '{path}' not found" class ResourceInvalid(ResourceError): - """Resource has the wrong type. - """ + """Resource has the wrong type.""" default_message = "resource '{path}' is invalid for this operation" class FileExists(ResourceError): - """File already exists. - """ + """File already exists.""" default_message = "resource '{path}' exists" class FileExpected(ResourceInvalid): - """Operation only works on files. - """ + """Operation only works on files.""" default_message = "path '{path}' should be a file" class DirectoryExpected(ResourceInvalid): - """Operation only works on directories. - """ + """Operation only works on directories.""" default_message = "path '{path}' should be a directory" class DestinationExists(ResourceError): - """Target destination already exists. - """ + """Target destination already exists.""" default_message = "destination '{path}' exists" class DirectoryExists(ResourceError): - """Directory already exists. - """ + """Directory already exists.""" default_message = "directory '{path}' exists" class DirectoryNotEmpty(ResourceError): - """Attempt to remove a non-empty directory. - """ + """Attempt to remove a non-empty directory.""" default_message = "directory '{path}' is not empty" class ResourceLocked(ResourceError): - """Attempt to use a locked resource. - """ + """Attempt to use a locked resource.""" default_message = "resource '{path}' is locked" class ResourceReadOnly(ResourceError): - """Attempting to modify a read-only resource. - """ + """Attempting to modify a read-only resource.""" default_message = "resource '{path}' is read only" diff --git a/fs/ftpfs.py b/fs/ftpfs.py index e3a39411..0ba360ed 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -403,8 +403,7 @@ def host(self): @classmethod def _parse_features(cls, feat_response): # type: (Text) -> Dict[Text, Text] - """Parse a dict of features from FTP feat response. - """ + """Parse a dict of features from FTP feat response.""" features = {} if feat_response.split("-")[0] == "211": for line in feat_response.splitlines(): @@ -415,8 +414,7 @@ def _parse_features(cls, feat_response): def _open_ftp(self): # type: () -> FTP - """Open a new ftp object. - """ + """Open a new ftp object.""" _ftp = FTP() _ftp.set_debuglevel(0) with ftp_errors(self): @@ -462,8 +460,7 @@ def ftp_url(self): @property def ftp(self): # type: () -> FTP - """~ftplib.FTP: the underlying FTP client. - """ + """~ftplib.FTP: the underlying FTP client.""" return self._get_ftp() def geturl(self, path, purpose="download"): @@ -483,8 +480,7 @@ def _get_ftp(self): @property def features(self): # type: () -> Dict[Text, Text] - """dict: features of the remote FTP server. - """ + """dict: features of the remote FTP server.""" self._get_ftp() return self._features @@ -506,8 +502,7 @@ def _read_dir(self, path): @property def supports_mlst(self): # type: () -> bool - """bool: whether the server supports MLST feature. - """ + """bool: whether the server supports MLST feature.""" return "MLST" in self.features def create(self, path, wipe=False): @@ -525,8 +520,7 @@ def create(self, path, wipe=False): @classmethod def _parse_ftp_time(cls, time_text): # type: (Text) -> Optional[int] - """Parse a time from an ftp directory listing. - """ + """Parse a time from an ftp directory listing.""" try: tm_year = int(time_text[0:4]) tm_month = int(time_text[4:6]) diff --git a/fs/glob.py b/fs/glob.py index ac12125e..ffbc75dc 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -94,16 +94,16 @@ def imatch(pattern, path): class Globber(object): """A generator of glob results. - Arguments: - fs (~fs.base.FS): A filesystem object - pattern (str): A glob pattern, e.g. ``"**/*.py"`` - path (str): A path to a directory in the filesystem. - namespaces (list): A list of additional info namespaces. - case_sensitive (bool): If ``True``, the path matching will be - case *sensitive* i.e. ``"FOO.py"`` and ``"foo.py"`` will - be different, otherwise path matching will be case *insensitive*. - exclude_dirs (list): A list of patterns to exclude when searching, - e.g. ``["*.git"]``. + Arguments: + fs (~fs.base.FS): A filesystem object + pattern (str): A glob pattern, e.g. ``"**/*.py"`` + path (str): A path to a directory in the filesystem. + namespaces (list): A list of additional info namespaces. + case_sensitive (bool): If ``True``, the path matching will be + case *sensitive* i.e. ``"FOO.py"`` and ``"foo.py"`` will + be different, otherwise path matching will be case *insensitive*. + exclude_dirs (list): A list of patterns to exclude when searching, + e.g. ``["*.git"]``. """ diff --git a/fs/info.py b/fs/info.py index 13f7b17f..266a109e 100644 --- a/fs/info.py +++ b/fs/info.py @@ -49,8 +49,7 @@ class Info(object): def __init__(self, raw_info, to_datetime=epoch_to_datetime): # type: (RawInfo, ToDatetime) -> None - """Create a resource info object from a raw info dict. - """ + """Create a resource info object from a raw info dict.""" self.raw = raw_info self._to_datetime = to_datetime self.namespaces = frozenset(self.raw.keys()) @@ -160,8 +159,7 @@ def has_namespace(self, namespace): def copy(self, to_datetime=None): # type: (Optional[ToDatetime]) -> Info - """Create a copy of this resource info object. - """ + """Create a copy of this resource info object.""" return Info(deepcopy(self.raw), to_datetime=to_datetime or self._to_datetime) def make_path(self, dir_path): @@ -180,8 +178,7 @@ def make_path(self, dir_path): @property def name(self): # type: () -> Text - """`str`: the resource name. - """ + """`str`: the resource name.""" return cast(Text, self.get("basic", "name")) @property @@ -238,22 +235,19 @@ def stem(self): @property def is_dir(self): # type: () -> bool - """`bool`: `True` if the resource references a directory. - """ + """`bool`: `True` if the resource references a directory.""" return cast(bool, self.get("basic", "is_dir")) @property def is_file(self): # type: () -> bool - """`bool`: `True` if the resource references a file. - """ + """`bool`: `True` if the resource references a file.""" return not cast(bool, self.get("basic", "is_dir")) @property def is_link(self): # type: () -> bool - """`bool`: `True` if the resource is a symlink. - """ + """`bool`: `True` if the resource is a symlink.""" self._require_namespace("link") return self.get("link", "target", None) is not None diff --git a/fs/iotools.py b/fs/iotools.py index 44849680..ed3bc9a1 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -25,8 +25,7 @@ class RawWrapper(io.RawIOBase): - """Convert a Python 2 style file-like object in to a IO object. - """ + """Convert a Python 2 style file-like object in to a IO object.""" def __init__(self, f, mode=None, name=None): # type: (IO[bytes], Optional[Text], Optional[Text]) -> None @@ -161,8 +160,7 @@ def make_stream( **kwargs # type: Any ): # type: (...) -> IO - """Take a Python 2.x binary file and return an IO Stream. - """ + """Take a Python 2.x binary file and return an IO Stream.""" reading = "r" in mode writing = "w" in mode appending = "a" in mode diff --git a/fs/lrucache.py b/fs/lrucache.py index 490d2700..11858416 100644 --- a/fs/lrucache.py +++ b/fs/lrucache.py @@ -27,8 +27,7 @@ def __init__(self, cache_size): def __setitem__(self, key, value): # type: (_K, _V) -> None - """Store a new views, potentially discarding an old value. - """ + """Store a new views, potentially discarding an old value.""" if key not in self: if len(self) >= self.cache_size: self.popitem(last=False) @@ -36,8 +35,7 @@ def __setitem__(self, key, value): def __getitem__(self, key): # type: (_K) -> _V - """Get the item, but also makes it most recent. - """ + """Get the item, but also makes it most recent.""" _super = typing.cast(OrderedDict, super(LRUCache, self)) value = _super.__getitem__(key) _super.__delitem__(key) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index d1c23724..63925d85 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -90,14 +90,12 @@ def _seek_lock(self): def on_modify(self): # noqa: D401 # type: () -> None - """Called when file data is modified. - """ + """Called when file data is modified.""" self._dir_entry.modified_time = self.modified_time = time.time() def on_access(self): # noqa: D401 # type: () -> None - """Called when file is accessed. - """ + """Called when file is accessed.""" self._dir_entry.accessed_time = self.accessed_time = time.time() def flush(self): @@ -326,8 +324,7 @@ class MemoryFS(FS): def __init__(self): # type: () -> None - """Create an in-memory filesystem. - """ + """Create an in-memory filesystem.""" self._meta = self._meta.copy() self.root = self._make_dir_entry(ResourceType.directory, "") super(MemoryFS, self).__init__() @@ -346,8 +343,7 @@ def _make_dir_entry(self, resource_type, name): def _get_dir_entry(self, dir_path): # type: (Text) -> Optional[_DirEntry] - """Get a directory entry, or `None` if one doesn't exist. - """ + """Get a directory entry, or `None` if one doesn't exist.""" with self._lock: dir_path = normpath(dir_path) current_entry = self.root # type: Optional[_DirEntry] diff --git a/fs/mode.py b/fs/mode.py index 16d51875..35018755 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -65,8 +65,7 @@ def __str__(self): def __contains__(self, character): # type: (object) -> bool - """Check if a mode contains a given character. - """ + """Check if a mode contains a given character.""" assert isinstance(character, Text) return character in self._mode @@ -123,64 +122,55 @@ def validate_bin(self): @property def create(self): # type: () -> bool - """`bool`: `True` if the mode would create a file. - """ + """`bool`: `True` if the mode would create a file.""" return "a" in self or "w" in self or "x" in self @property def reading(self): # type: () -> bool - """`bool`: `True` if the mode permits reading. - """ + """`bool`: `True` if the mode permits reading.""" return "r" in self or "+" in self @property def writing(self): # type: () -> bool - """`bool`: `True` if the mode permits writing. - """ + """`bool`: `True` if the mode permits writing.""" return "w" in self or "a" in self or "+" in self or "x" in self @property def appending(self): # type: () -> bool - """`bool`: `True` if the mode permits appending. - """ + """`bool`: `True` if the mode permits appending.""" return "a" in self @property def updating(self): # type: () -> bool - """`bool`: `True` if the mode permits both reading and writing. - """ + """`bool`: `True` if the mode permits both reading and writing.""" return "+" in self @property def truncate(self): # type: () -> bool - """`bool`: `True` if the mode would truncate an existing file. - """ + """`bool`: `True` if the mode would truncate an existing file.""" return "w" in self or "x" in self @property def exclusive(self): # type: () -> bool - """`bool`: `True` if the mode require exclusive creation. - """ + """`bool`: `True` if the mode require exclusive creation.""" return "x" in self @property def binary(self): # type: () -> bool - """`bool`: `True` if a mode specifies binary. - """ + """`bool`: `True` if a mode specifies binary.""" return "b" in self @property def text(self): # type: () -> bool - """`bool`: `True` if a mode specifies text. - """ + """`bool`: `True` if a mode specifies text.""" return "t" in self or "b" not in self diff --git a/fs/mountfs.py b/fs/mountfs.py index d51d7d9d..e0a4e956 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -41,8 +41,7 @@ class MountError(Exception): - """Thrown when mounts conflict. - """ + """Thrown when mounts conflict.""" class MountFS(FS): diff --git a/fs/multifs.py b/fs/multifs.py index e68d2c00..3e6abb13 100644 --- a/fs/multifs.py +++ b/fs/multifs.py @@ -127,14 +127,12 @@ def get_fs(self, name): def _resort(self): # type: () -> None - """Force `iterate_fs` to re-sort on next reference. - """ + """Force `iterate_fs` to re-sort on next reference.""" self._fs_sequence = None def iterate_fs(self): # type: () -> Iterator[Tuple[Text, FS]] - """Get iterator that returns (name, fs) in priority order. - """ + """Get iterator that returns (name, fs) in priority order.""" if self._fs_sequence is None: self._fs_sequence = [ (name, fs) @@ -146,8 +144,7 @@ def iterate_fs(self): def _delegate(self, path): # type: (Text) -> Optional[FS] - """Get a filesystem which has a given path. - """ + """Get a filesystem which has a given path.""" for _name, fs in self.iterate_fs(): if fs.exists(path): return fs @@ -155,8 +152,7 @@ def _delegate(self, path): def _delegate_required(self, path): # type: (Text) -> FS - """Check that there is a filesystem with the given ``path``. - """ + """Check that there is a filesystem with the given ``path``.""" fs = self._delegate(path) if fs is None: raise errors.ResourceNotFound(path) @@ -164,8 +160,7 @@ def _delegate_required(self, path): def _writable_required(self, path): # type: (Text) -> FS - """Check that ``path`` is writeable. - """ + """Check that ``path`` is writeable.""" if self.write_fs is None: raise errors.ResourceReadOnly(path) return self.write_fs diff --git a/fs/opener/appfs.py b/fs/opener/appfs.py index fccf603e..0b1d78fa 100644 --- a/fs/opener/appfs.py +++ b/fs/opener/appfs.py @@ -21,8 +21,7 @@ @registry.install class AppFSOpener(Opener): - """``AppFS`` opener. - """ + """``AppFS`` opener.""" protocols = ["userdata", "userconf", "sitedata", "siteconf", "usercache", "userlog"] _protocol_mapping = None diff --git a/fs/opener/errors.py b/fs/opener/errors.py index 593eb168..7c8ae8a5 100644 --- a/fs/opener/errors.py +++ b/fs/opener/errors.py @@ -4,25 +4,20 @@ class ParseError(ValueError): - """Attempt to parse an invalid FS URL. - """ + """Attempt to parse an invalid FS URL.""" class OpenerError(Exception): - """Base exception for opener related errors. - """ + """Base exception for opener related errors.""" class UnsupportedProtocol(OpenerError): - """No opener found for the given protocol. - """ + """No opener found for the given protocol.""" class EntryPointError(OpenerError): - """An entry point could not be loaded. - """ + """An entry point could not be loaded.""" class NotWriteable(OpenerError): - """A writable FS could not be created. - """ + """A writable FS could not be created.""" diff --git a/fs/opener/ftpfs.py b/fs/opener/ftpfs.py index f5beab21..af64606b 100644 --- a/fs/opener/ftpfs.py +++ b/fs/opener/ftpfs.py @@ -21,8 +21,7 @@ @registry.install class FTPOpener(Opener): - """`FTPFS` opener. - """ + """`FTPFS` opener.""" protocols = ["ftp"] diff --git a/fs/opener/memoryfs.py b/fs/opener/memoryfs.py index 696ee06a..1ce8f105 100644 --- a/fs/opener/memoryfs.py +++ b/fs/opener/memoryfs.py @@ -19,8 +19,7 @@ @registry.install class MemOpener(Opener): - """`MemoryFS` opener. - """ + """`MemoryFS` opener.""" protocols = ["mem"] diff --git a/fs/opener/osfs.py b/fs/opener/osfs.py index 00cb63ee..7cb87b99 100644 --- a/fs/opener/osfs.py +++ b/fs/opener/osfs.py @@ -19,8 +19,7 @@ @registry.install class OSFSOpener(Opener): - """`OSFS` opener. - """ + """`OSFS` opener.""" protocols = ["file", "osfs"] diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 50f2976c..d01530dc 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -31,8 +31,7 @@ class Registry(object): - """A registry for `Opener` instances. - """ + """A registry for `Opener` instances.""" def __init__(self, default_opener="osfs", load_extern=False): # type: (Text, bool) -> None @@ -64,6 +63,7 @@ def install(self, opener): Note: May be used as a class decorator. For example:: + registry = Registry() @registry.install class ArchiveOpener(Opener): @@ -79,8 +79,7 @@ class ArchiveOpener(Opener): @property def protocols(self): # type: () -> List[Text] - """`list`: the list of supported protocols. - """ + """`list`: the list of supported protocols.""" _protocols = list(self._protocols) if self.load_extern: diff --git a/fs/opener/tarfs.py b/fs/opener/tarfs.py index 3ff91f55..bacb4e65 100644 --- a/fs/opener/tarfs.py +++ b/fs/opener/tarfs.py @@ -20,8 +20,7 @@ @registry.install class TarOpener(Opener): - """`TarFS` opener. - """ + """`TarFS` opener.""" protocols = ["tar"] diff --git a/fs/opener/tempfs.py b/fs/opener/tempfs.py index ffa17983..22e26e0c 100644 --- a/fs/opener/tempfs.py +++ b/fs/opener/tempfs.py @@ -19,8 +19,7 @@ @registry.install class TempOpener(Opener): - """`TempFS` opener. - """ + """`TempFS` opener.""" protocols = ["temp"] diff --git a/fs/opener/zipfs.py b/fs/opener/zipfs.py index 81e48455..dbc0fe7c 100644 --- a/fs/opener/zipfs.py +++ b/fs/opener/zipfs.py @@ -20,8 +20,7 @@ @registry.install class ZipOpener(Opener): - """`ZipFS` opener. - """ + """`ZipFS` opener.""" protocols = ["zip"] diff --git a/fs/osfs.py b/fs/osfs.py index f854b16a..ed091e64 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -112,8 +112,7 @@ def __init__( expand_vars=True, # type: bool ): # type: (...) -> None - """Create an OSFS instance. - """ + """Create an OSFS instance.""" super(OSFS, self).__init__() if isinstance(root_path, bytes): root_path = fsdecode(root_path) @@ -188,8 +187,7 @@ def __str__(self): def _to_sys_path(self, path): # type: (Text) -> bytes - """Convert a FS path to a path on the OS. - """ + """Convert a FS path to a path on the OS.""" sys_path = fsencode( os.path.join(self._root_path, path.lstrip("/").replace("/", os.sep)) ) @@ -198,8 +196,7 @@ def _to_sys_path(self, path): @classmethod def _make_details_from_stat(cls, stat_result): # type: (os.stat_result) -> Dict[Text, object] - """Make a *details* info dict from an `os.stat_result` object. - """ + """Make a *details* info dict from an `os.stat_result` object.""" details = { "_write": ["accessed", "modified"], "accessed": stat_result.st_atime, @@ -218,8 +215,7 @@ def _make_details_from_stat(cls, stat_result): @classmethod def _make_access_from_stat(cls, stat_result): # type: (os.stat_result) -> Dict[Text, object] - """Make an *access* info dict from an `os.stat_result` object. - """ + """Make an *access* info dict from an `os.stat_result` object.""" access = {} # type: Dict[Text, object] access["permissions"] = Permissions(mode=stat_result.st_mode).dump() access["gid"] = gid = stat_result.st_gid @@ -252,8 +248,7 @@ def _make_access_from_stat(cls, stat_result): @classmethod def _get_type_from_stat(cls, _stat): # type: (os.stat_result) -> ResourceType - """Get the resource type from an `os.stat_result` object. - """ + """Get the resource type from an `os.stat_result` object.""" st_mode = _stat.st_mode st_type = stat.S_IFMT(st_mode) return cls.STAT_TO_RESOURCE_TYPE.get(st_type, ResourceType.unknown) @@ -673,6 +668,6 @@ def validatepath(self, path): raise errors.InvalidCharsInPath( path, msg="path '{path}' could not be encoded for the filesystem (check LANG" - " env var); {error}".format(path=path, error=error), + " env var); {error}".format(path=path, error=error), ) return super(OSFS, self).validatepath(path) diff --git a/fs/permissions.py b/fs/permissions.py index 19934465..7e340f89 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -18,14 +18,12 @@ def make_mode(init): # type: (Union[int, Iterable[Text], None]) -> int - """Make a mode integer from an initial value. - """ + """Make a mode integer from an initial value.""" return Permissions.get_mode(init) class _PermProperty(object): - """Creates simple properties to get/set permissions. - """ + """Creates simple properties to get/set permissions.""" def __init__(self, name): # type: (Text) -> None @@ -174,8 +172,7 @@ def __ne__(self, other): @classmethod def parse(cls, ls): # type: (Text) -> Permissions - """Parse permissions in Linux notation. - """ + """Parse permissions in Linux notation.""" user = ls[:3] group = ls[3:6] other = ls[6:9] @@ -184,8 +181,7 @@ def parse(cls, ls): @classmethod def load(cls, permissions): # type: (List[Text]) -> Permissions - """Load a serialized permissions object. - """ + """Load a serialized permissions object.""" return cls(names=permissions) @classmethod @@ -222,26 +218,22 @@ def create(cls, init=None): @classmethod def get_mode(cls, init): # type: (Union[int, Iterable[Text], None]) -> int - """Convert an initial value to a mode integer. - """ + """Convert an initial value to a mode integer.""" return cls.create(init).mode def copy(self): # type: () -> Permissions - """Make a copy of this permissions object. - """ + """Make a copy of this permissions object.""" return Permissions(names=list(self._perms)) def dump(self): # type: () -> List[Text] - """Get a list suitable for serialization. - """ + """Get a list suitable for serialization.""" return sorted(self._perms) def as_str(self): # type: () -> Text - """Get a Linux-style string representation of permissions. - """ + """Get a Linux-style string representation of permissions.""" perms = [ c if name in self._perms else "-" for name, c in zip(self._LINUX_PERMS_NAMES[-9:], "rwxrwxrwx") @@ -259,8 +251,7 @@ def as_str(self): @property def mode(self): # type: () -> int - """`int`: mode integer. - """ + """`int`: mode integer.""" mode = 0 for name, mask in self._LINUX_PERMS: if name in self._perms: diff --git a/fs/subfs.py b/fs/subfs.py index 7172008e..81e19c9d 100644 --- a/fs/subfs.py +++ b/fs/subfs.py @@ -55,8 +55,7 @@ def delegate_path(self, path): class ClosingSubFS(SubFS[_F], typing.Generic[_F]): - """A version of `SubFS` which closes its parent when closed. - """ + """A version of `SubFS` which closes its parent when closed.""" def close(self): # type: () -> None diff --git a/fs/tarfs.py b/fs/tarfs.py index 4f48d821..b9b14c0a 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -157,8 +157,7 @@ def __init__( @six.python_2_unicode_compatible class WriteTarFS(WrapFS): - """A writable tar file. - """ + """A writable tar file.""" def __init__( self, @@ -234,8 +233,7 @@ def write_tar( @six.python_2_unicode_compatible class ReadTarFS(FS): - """A readable tar file. - """ + """A readable tar file.""" _meta = { "case_insensitive": True, diff --git a/fs/tempfs.py b/fs/tempfs.py index 748463c1..33a12e91 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -76,8 +76,7 @@ def close(self): def clean(self): # type: () -> None - """Clean (delete) temporary files created by this filesystem. - """ + """Clean (delete) temporary files created by this filesystem.""" if self._cleaned: return diff --git a/fs/test.py b/fs/test.py index de70f280..4c630137 100644 --- a/fs/test.py +++ b/fs/test.py @@ -245,13 +245,10 @@ class FSTestCases(object): - """Basic FS tests. - """ + """Basic FS tests.""" def make_fs(self): - """Return an FS instance. - - """ + """Return an FS instance.""" raise NotImplementedError("implement me") def destroy_fs(self, fs): @@ -430,15 +427,13 @@ def test_geturl(self): self.fs.hasurl("a/b/c/foo/bar") def test_geturl_purpose(self): - """Check an unknown purpose raises a NoURL error. - """ + """Check an unknown purpose raises a NoURL error.""" self.fs.create("foo") with self.assertRaises(errors.NoURL): self.fs.geturl("foo", purpose="__nosuchpurpose__") def test_validatepath(self): - """Check validatepath returns an absolute path. - """ + """Check validatepath returns an absolute path.""" path = self.fs.validatepath("foo") self.assertEqual(path, "/foo") @@ -1494,7 +1489,7 @@ def test_upload(self): with self.fs.open("foo", "rb") as f: data = f.read() self.assertEqual(data, b"bar") - + # upload to non-existing path (/spam/eggs) with self.assertRaises(errors.ResourceNotFound): self.fs.upload("/spam/eggs", bytes_file) diff --git a/fs/time.py b/fs/time.py index 5af60578..f1638aa3 100644 --- a/fs/time.py +++ b/fs/time.py @@ -16,13 +16,11 @@ def datetime_to_epoch(d): # type: (datetime) -> int - """Convert datetime to epoch. - """ + """Convert datetime to epoch.""" return timegm(d.utctimetuple()) def epoch_to_datetime(t): # type: (int) -> datetime - """Convert epoch time to a UTC datetime. - """ + """Convert epoch time to a UTC datetime.""" return utclocalize(utcfromtimestamp(t)) if t is not None else None diff --git a/fs/tree.py b/fs/tree.py index faee5472..984ea353 100644 --- a/fs/tree.py +++ b/fs/tree.py @@ -79,8 +79,7 @@ def render( def write(line): # type: (Text) -> None - """Write a line to the output. - """ + """Write a line to the output.""" print(line, file=file) # FIXME(@althonos): define functions using `with_color` and @@ -88,32 +87,28 @@ def write(line): def format_prefix(prefix): # type: (Text) -> Text - """Format the prefix lines. - """ + """Format the prefix lines.""" if not with_color: return prefix return "\x1b[32m%s\x1b[0m" % prefix def format_dirname(dirname): # type: (Text) -> Text - """Format a directory name. - """ + """Format a directory name.""" if not with_color: return dirname return "\x1b[1;34m%s\x1b[0m" % dirname def format_error(msg): # type: (Text) -> Text - """Format an error. - """ + """Format an error.""" if not with_color: return msg return "\x1b[31m%s\x1b[0m" % msg def format_filename(fname): # type: (Text) -> Text - """Format a filename. - """ + """Format a filename.""" if not with_color: return fname if fname.startswith("."): @@ -122,22 +117,19 @@ def format_filename(fname): def sort_key_dirs_first(info): # type: (Info) -> Tuple[bool, Text] - """Get the info sort function with directories first. - """ + """Get the info sort function with directories first.""" return (not info.is_dir, info.name.lower()) def sort_key(info): # type: (Info) -> Text - """Get the default info sort function using resource name. - """ + """Get the default info sort function using resource name.""" return info.name.lower() counts = {"dirs": 0, "files": 0} def format_directory(path, levels): # type: (Text, List[bool]) -> None - """Recursive directory function. - """ + """Recursive directory function.""" try: directory = sorted( fs.filterdir(path, exclude_dirs=exclude, files=filter), diff --git a/fs/walk.py b/fs/walk.py index 3e44537d..44a910d1 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -198,8 +198,7 @@ def _iter_walk( def _check_open_dir(self, fs, path, info): # type: (FS, Text, Info) -> bool - """Check if a directory should be considered in the walk. - """ + """Check if a directory should be considered in the walk.""" if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name): return False if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name): @@ -411,8 +410,7 @@ def _walk_breadth( namespaces=None, # type: Optional[Collection[Text]] ): # type: (...) -> Iterator[Tuple[Text, Optional[Info]]] - """Walk files using a *breadth first* search. - """ + """Walk files using a *breadth first* search.""" queue = deque([path]) push = queue.appendleft pop = queue.pop @@ -447,8 +445,7 @@ def _walk_depth( namespaces=None, # type: Optional[Collection[Text]] ): # type: (...) -> Iterator[Tuple[Text, Optional[Info]]] - """Walk files using a *depth first* search. - """ + """Walk files using a *depth first* search.""" # No recursion! _combine = combine @@ -526,8 +523,7 @@ def __repr__(self): def _make_walker(self, *args, **kwargs): # type: (*Any, **Any) -> Walker - """Create a walker instance. - """ + """Create a walker instance.""" walker = self.walker_class(*args, **kwargs) return walker diff --git a/fs/wildcard.py b/fs/wildcard.py index 6c710cad..a43a84b7 100644 --- a/fs/wildcard.py +++ b/fs/wildcard.py @@ -32,7 +32,7 @@ def match(pattern, name): try: re_pat = _PATTERN_CACHE[(pattern, True)] except KeyError: - res = "(?ms)" + _translate(pattern) + r'\Z' + res = "(?ms)" + _translate(pattern) + r"\Z" _PATTERN_CACHE[(pattern, True)] = re_pat = re.compile(res) return re_pat.match(name) is not None @@ -52,7 +52,7 @@ def imatch(pattern, name): try: re_pat = _PATTERN_CACHE[(pattern, False)] except KeyError: - res = "(?ms)" + _translate(pattern, case_sensitive=False) + r'\Z' + res = "(?ms)" + _translate(pattern, case_sensitive=False) + r"\Z" _PATTERN_CACHE[(pattern, False)] = re_pat = re.compile(res, re.IGNORECASE) return re_pat.match(name) is not None diff --git a/fs/zipfs.py b/fs/zipfs.py index 8feb9e56..32d0efcc 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -198,8 +198,7 @@ def __init__( @six.python_2_unicode_compatible class WriteZipFS(WrapFS): - """A writable zip file. - """ + """A writable zip file.""" def __init__( self, @@ -276,8 +275,7 @@ def write_zip( @six.python_2_unicode_compatible class ReadZipFS(FS): - """A readable zip file. - """ + """A readable zip file.""" _meta = { "case_insensitive": True, @@ -308,8 +306,7 @@ def __str__(self): def _path_to_zip_name(self, path): # type: (Text) -> str - """Convert a path to a zip file name. - """ + """Convert a path to a zip file name.""" path = relpath(normpath(path)) if self._directory.isdir(path): path = forcedir(path) @@ -320,8 +317,7 @@ def _path_to_zip_name(self, path): @property def _directory(self): # type: () -> MemoryFS - """`MemoryFS`: a filesystem with the same folder hierarchy as the zip. - """ + """`MemoryFS`: a filesystem with the same folder hierarchy as the zip.""" self.check() with self._lock: if self._directory_fs is None: diff --git a/tests/test_appfs.py b/tests/test_appfs.py index d673c9b2..871526f3 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -33,7 +33,7 @@ def make_fs(self): "appdirs.{}".format(self.AppFS.app_dir), autospec=True, spec_set=True, - return_value=tempfile.mkdtemp(dir=self.tmpdir) + return_value=tempfile.mkdtemp(dir=self.tmpdir), ): return self.AppFS("fstest", "willmcgugan", "1.0") @@ -42,7 +42,9 @@ def make_fs(self): def test_repr(self): self.assertEqual( repr(self.fs), - "{}(u'fstest', author=u'willmcgugan', version=u'1.0')".format(self.AppFS.__name__) + "{}(u'fstest', author=u'willmcgugan', version=u'1.0')".format( + self.AppFS.__name__ + ), ) else: @@ -50,31 +52,36 @@ def test_repr(self): def test_repr(self): self.assertEqual( repr(self.fs), - "{}('fstest', author='willmcgugan', version='1.0')".format(self.AppFS.__name__) + "{}('fstest', author='willmcgugan', version='1.0')".format( + self.AppFS.__name__ + ), ) - def test_str(self): self.assertEqual( - str(self.fs), - "<{} 'fstest'>".format(self.AppFS.__name__.lower()) + str(self.fs), "<{} 'fstest'>".format(self.AppFS.__name__.lower()) ) class TestUserDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.UserDataFS + class TestUserConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.UserConfigFS + class TestUserCacheFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.UserCacheFS + class TestSiteDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.SiteDataFS + class TestSiteConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.SiteConfigFS + class TestUserLogFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): AppFS = appfs.UserLogFS diff --git a/tests/test_encoding.py b/tests/test_encoding.py index c8782ff2..0cd91d4c 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -14,10 +14,7 @@ if platform.system() != "Windows": - @unittest.skipIf( - platform.system() == "Darwin", - "Bad unicode not possible on OSX" - ) + @unittest.skipIf(platform.system() == "Darwin", "Bad unicode not possible on OSX") class TestEncoding(unittest.TestCase): TEST_FILENAME = b"foo\xb1bar" diff --git a/tests/test_errors.py b/tests/test_errors.py index 1ed98c54..5f4d8b8c 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -30,7 +30,7 @@ def test_raise_in_multiprocessing(self): [errors.NoURL, "some_path", "some_purpose"], [errors.Unsupported], [errors.IllegalBackReference, "path"], - [errors.MissingInfoNamespace, "path"] + [errors.MissingInfoNamespace, "path"], ] try: pool = multiprocessing.Pool(1) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 530e020e..a546bbdd 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -29,11 +29,11 @@ try: from pytest.mark import slow except: + def slow(cls): return cls - # Prevent socket timeouts from slowing tests too much socket.setdefaulttimeout(1) diff --git a/tests/test_info.py b/tests/test_info.py index 8c5a1d30..f83c1e7b 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,4 +1,3 @@ - from __future__ import unicode_literals import datetime diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 842c41fb..0b26a576 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -29,8 +29,7 @@ def _create_many_files(self): ) @unittest.skipUnless( - tracemalloc, - reason="`tracemalloc` isn't supported on this Python version." + tracemalloc, reason="`tracemalloc` isn't supported on this Python version." ) def test_close_mem_free(self): """Ensure all file memory is freed when calling close(). diff --git a/tests/test_move.py b/tests/test_move.py index bec4d776..d87d2bd6 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -1,4 +1,3 @@ - from __future__ import unicode_literals import unittest diff --git a/tests/test_opener.py b/tests/test_opener.py index b01e8d4e..fde555a2 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -208,7 +208,6 @@ def test_manage_fs_error(self): class TestOpeners(unittest.TestCase): - def setUp(self): self.tmpdir = tempfile.mkdtemp() diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index c518e8c4..2a03afec 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -236,8 +236,7 @@ def test_listdir(self): class TestImplicitDirectories(unittest.TestCase): - """Regression tests for #160. - """ + """Regression tests for #160.""" @classmethod def setUpClass(cls): diff --git a/tox.ini b/tox.ini index ec1d7037..3d57773a 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ deps = python = python3.9 commands = black --check {toxinidir}/fs deps = - black + black==20.8b1 [testenv:pydocstyle] python = python3.9 From 5a27e2e29df780ecbabf92d31d44b7f3ce271918 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 12:35:31 +0100 Subject: [PATCH 081/309] Fix type-related issues in `ftpfs.FTPFile` and add support for `array` arguments --- fs/ftpfs.py | 23 ++++++++++++++++------- fs/opener/parse.py | 12 ++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 0ba360ed..4a3f9524 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -4,6 +4,7 @@ from __future__ import print_function from __future__ import unicode_literals +import array import calendar import io import itertools @@ -36,6 +37,7 @@ from . import _ftp_parse as ftp_parse if typing.TYPE_CHECKING: + import mmap import ftplib from typing import ( Any, @@ -236,14 +238,17 @@ def read(self, size=-1): return b"".join(chunks) def readinto(self, buffer): - # type: (bytearray) -> int + # type: (Union[bytearray, memoryview, array.array[Any], mmap.mmap]) -> int data = self.read(len(buffer)) bytes_read = len(data) - buffer[:bytes_read] = data + if isinstance(buffer, array.array): + buffer[:bytes_read] = array.array(buffer.typecode, data) + else: + buffer[:bytes_read] = data # type: ignore return bytes_read - def readline(self, size=-1): - # type: (int) -> bytes + def readline(self, size=None): + # type: (Optional[int]) -> bytes return next(line_iterator(self, size)) # type: ignore def readlines(self, hint=-1): @@ -262,10 +267,13 @@ def writable(self): return self.mode.writing def write(self, data): - # type: (bytes) -> int + # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int if not self.mode.writing: raise IOError("File not open for writing") + if isinstance(data, array.array): + data = data.tobytes() + with self._lock: conn = self.write_conn data_pos = 0 @@ -281,8 +289,9 @@ def write(self, data): return data_pos def writelines(self, lines): - # type: (Iterable[bytes]) -> None - self.write(b"".join(lines)) + # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None + for line in lines: + self.write(line) def truncate(self, size=None): # type: (Optional[int]) -> int diff --git a/fs/opener/parse.py b/fs/opener/parse.py index e9423807..e49a8009 100644 --- a/fs/opener/parse.py +++ b/fs/opener/parse.py @@ -18,12 +18,12 @@ from typing import Optional, Text -_ParseResult = collections.namedtuple( - "ParseResult", ["protocol", "username", "password", "resource", "params", "path"] -) - - -class ParseResult(_ParseResult): +class ParseResult( + collections.namedtuple( + "ParseResult", + ["protocol", "username", "password", "resource", "params", "path"], + ) +): """A named tuple containing fields of a parsed FS URL. Attributes: From 221b9e53a761353425c73d3babe565b89f78d8aa Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 12:43:43 +0100 Subject: [PATCH 082/309] Rewrite buffering in `FTPFile.writelines` to work with more argument types --- fs/ftpfs.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 4a3f9524..9c0cf95e 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -290,8 +290,15 @@ def write(self, data): def writelines(self, lines): # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None + if not self.mode.writing: + raise IOError("File not open for writing") + data = bytearray() for line in lines: - self.write(line) + if isinstance(line, array.array): + data.extend(line.tobytes()) + else: + data.extend(line) # type: ignore + self.write(data) def truncate(self, size=None): # type: (Optional[int]) -> int From ec8300b3dac2bfa68d59b3b290e40c3fb608976b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 13:10:55 +0100 Subject: [PATCH 083/309] Fix typing of some methods in `fs.iotools` and `fs.memoryfs` file APIs --- fs/iotools.py | 30 +++++++++++++++++++----------- fs/memoryfs.py | 15 +++++++++------ fs/tree.py | 2 +- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/fs/iotools.py b/fs/iotools.py index ed3bc9a1..82f827ce 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -4,6 +4,7 @@ from __future__ import print_function from __future__ import unicode_literals +import array import io import typing from io import SEEK_SET, SEEK_CUR @@ -11,6 +12,7 @@ from .mode import Mode if typing.TYPE_CHECKING: + import mmap from io import RawIOBase from typing import ( Any, @@ -88,8 +90,11 @@ def truncate(self, size=None): return self._f.truncate(size) def write(self, data): - # type: (bytes) -> int - count = self._f.write(data) + # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int + if isinstance(data, array.array): + count = self._f.write(data.tobytes()) + else: + count = self._f.write(data) # type: ignore return len(data) if count is None else count @typing.no_type_check @@ -130,17 +135,20 @@ def readinto1(self, b): b[:bytes_read] = data return bytes_read - def readline(self, limit=-1): - # type: (int) -> bytes - return self._f.readline(limit) + def readline(self, limit=None): + # type: (Optional[int]) -> bytes + return self._f.readline(-1 if limit is None else limit) - def readlines(self, hint=-1): - # type: (int) -> List[bytes] - return self._f.readlines(hint) + def readlines(self, hint=None): + # type: (Optional[int]) -> List[bytes] + return self._f.readlines(-1 if hint is None else hint) - def writelines(self, sequence): - # type: (Iterable[Union[bytes, bytearray]]) -> None - return self._f.writelines(sequence) + def writelines(self, lines): + # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None + _lines = ( + line.tobytes() if isinstance(line, array.array) else line for line in lines + ) + return self._f.writelines(typing.cast("Iterable[bytes]", _lines)) def __iter__(self): # type: () -> Iterator[bytes] diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 63925d85..0a5f52b6 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -24,11 +24,14 @@ from ._typing import overload if typing.TYPE_CHECKING: + import array + import mmap from typing import ( Any, BinaryIO, Collection, Dict, + Iterable, Iterator, List, Optional, @@ -116,8 +119,8 @@ def next(self): __next__ = next - def readline(self, size=-1): - # type: (int) -> bytes + def readline(self, size=None): + # type: (Optional[int]) -> bytes if not self._mode.reading: raise IOError("File not open for reading") with self._seek_lock(): @@ -131,7 +134,7 @@ def close(self): self._dir_entry.remove_open_file(self) super(_MemoryFile, self).close() - def read(self, size=-1): + def read(self, size=None): # type: (Optional[int]) -> bytes if not self._mode.reading: raise IOError("File not open for reading") @@ -190,15 +193,15 @@ def writable(self): return self._mode.writing def write(self, data): - # type: (bytes) -> int + # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int if not self._mode.writing: raise IOError("File not open for writing") with self._seek_lock(): self.on_modify() return self._bytes_io.write(data) - def writelines(self, sequence): # type: ignore - # type: (List[bytes]) -> None + def writelines(self, sequence): + # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None # FIXME(@althonos): For some reason the stub for IOBase.writelines # is List[Any] ?! It should probably be Iterable[ByteString] with self._seek_lock(): diff --git a/fs/tree.py b/fs/tree.py index 984ea353..0f3142fe 100644 --- a/fs/tree.py +++ b/fs/tree.py @@ -133,7 +133,7 @@ def format_directory(path, levels): try: directory = sorted( fs.filterdir(path, exclude_dirs=exclude, files=filter), - key=sort_key_dirs_first if dirs_first else sort_key, + key=sort_key_dirs_first if dirs_first else sort_key, # type: ignore ) except Exception as error: prefix = ( From 70ee97ab2e96861a61fbb61f330278417b92054e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 13:12:33 +0100 Subject: [PATCH 084/309] Relax Python version but pin dependencies in `tox` linter environments --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 3d57773a..23ce73fc 100644 --- a/tox.ini +++ b/tox.ini @@ -17,24 +17,22 @@ deps = !scandir: . [testenv:mypy] -basepython = python3.9 commands = mypy --config-file {toxinidir}/setup.cfg {toxinidir}/fs deps = . mypy==0.800 [testenv:flake8] -python = python3.9 commands = flake8 {toxinidir}/fs {toxinidir}/tests deps = - flake8 - flake8-bugbear - flake8-builtins - flake8-comprehensions - flake8-mutable + flake8==3.7.9 + #flake8-builtins==1.5.3 + flake8-bugbear==19.8.0 + flake8-comprehensions==3.1.4 + flake8-mutable==1.2.0 + flake8-tuple==0.4.0 [testenv:black] -python = python3.9 commands = black --check {toxinidir}/fs deps = black==20.8b1 From a433d9e8fd2c5be0059c7e1d8f545afea05809de Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 13:29:12 +0100 Subject: [PATCH 085/309] Fix `flake8` warnings in `fs` and `tests` modules --- fs/ftpfs.py | 4 ++-- fs/info.py | 6 +++--- fs/iotools.py | 4 ++-- fs/memoryfs.py | 18 ++++++++---------- tests/test_appfs.py | 1 - tests/test_ftpfs.py | 2 +- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 9c0cf95e..0ec93017 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -267,7 +267,7 @@ def writable(self): return self.mode.writing def write(self, data): - # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int + # type: (Union[bytes, memoryview, array.array[Any], mmap.mmap]) -> int if not self.mode.writing: raise IOError("File not open for writing") @@ -289,7 +289,7 @@ def write(self, data): return data_pos def writelines(self, lines): - # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None + # type: (Iterable[Union[bytes, memoryview, array.array[Any], mmap.mmap]]) -> None # noqa: E501 if not self.mode.writing: raise IOError("File not open for writing") data = bytearray() diff --git a/fs/info.py b/fs/info.py index 266a109e..e50801c9 100644 --- a/fs/info.py +++ b/fs/info.py @@ -72,8 +72,8 @@ def _make_datetime(self, t): # type: (None) -> None pass - @overload # noqa: F811 - def _make_datetime(self, t): + @overload + def _make_datetime(self, t): # noqa: F811 # type: (int) -> datetime pass @@ -90,7 +90,7 @@ def get(self, namespace, key): pass @overload # noqa: F811 - def get(self, namespace, key, default): + def get(self, namespace, key, default): # noqa: F811 # type: (Text, Text, T) -> Union[Any, T] pass diff --git a/fs/iotools.py b/fs/iotools.py index 82f827ce..aa1bb92e 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -90,7 +90,7 @@ def truncate(self, size=None): return self._f.truncate(size) def write(self, data): - # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int + # type: (Union[bytes, memoryview, array.array[Any], mmap.mmap]) -> int if isinstance(data, array.array): count = self._f.write(data.tobytes()) else: @@ -144,7 +144,7 @@ def readlines(self, hint=None): return self._f.readlines(-1 if hint is None else hint) def writelines(self, lines): - # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None + # type: (Iterable[Union[bytes, memoryview, array.array[Any], mmap.mmap]]) -> None # noqa: E501 _lines = ( line.tobytes() if isinstance(line, array.array) else line for line in lines ) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 0a5f52b6..3164d721 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -193,7 +193,7 @@ def writable(self): return self._mode.writing def write(self, data): - # type: (Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]) -> int + # type: (Union[bytes, memoryview, array.array[Any], mmap.mmap]) -> int if not self._mode.writing: raise IOError("File not open for writing") with self._seek_lock(): @@ -201,9 +201,7 @@ def write(self, data): return self._bytes_io.write(data) def writelines(self, sequence): - # type: (Iterable[Union[bytes, bytearray, memoryview, array.array[Any], mmap.mmap]]) -> None - # FIXME(@althonos): For some reason the stub for IOBase.writelines - # is List[Any] ?! It should probably be Iterable[ByteString] + # type: (Iterable[Union[bytes, memoryview, array.array[Any], mmap.mmap]]) -> None # noqa: E501 with self._seek_lock(): self.on_modify() self._bytes_io.writelines(sequence) @@ -248,18 +246,18 @@ def size(self): _bytes_file.seek(0, os.SEEK_END) return _bytes_file.tell() - @overload # noqa: F811 - def get_entry(self, name, default): + @overload + def get_entry(self, name, default): # noqa: F811 # type: (Text, _DirEntry) -> _DirEntry pass - @overload # noqa: F811 - def get_entry(self, name): + @overload + def get_entry(self, name): # noqa: F811 # type: (Text) -> Optional[_DirEntry] pass - @overload # noqa: F811 - def get_entry(self, name, default): + @overload + def get_entry(self, name, default): # noqa: F811 # type: (Text, None) -> Optional[_DirEntry] pass diff --git a/tests/test_appfs.py b/tests/test_appfs.py index 871526f3..9bd52be6 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import os import shutil import tempfile import unittest diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index a546bbdd..3709dc59 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -28,7 +28,7 @@ try: from pytest.mark import slow -except: +except ImportError: def slow(cls): return cls From afdcf8c3e9f683152144496110d14ff2ee17b38d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 14:09:02 +0100 Subject: [PATCH 086/309] Fix documentation-related errors found by `pydocstyle` --- fs/base.py | 30 ++++++++-------- fs/compress.py | 7 ++-- fs/copy.py | 4 +-- fs/errors.py | 18 +++++----- fs/ftpfs.py | 30 ++++++++-------- fs/glob.py | 51 +++++++++++++++------------- fs/info.py | 11 ++++-- fs/iotools.py | 2 +- fs/lrucache.py | 1 + fs/mirror.py | 1 + fs/mode.py | 15 ++++---- fs/mountfs.py | 12 ++++--- fs/move.py | 6 ++-- fs/multifs.py | 7 ++++ fs/opener/registry.py | 2 +- fs/permissions.py | 29 +++++++++------- fs/subfs.py | 2 +- fs/tarfs.py | 7 ++-- fs/tempfs.py | 26 +++++++------- fs/walk.py | 79 ++++++++++++++++++++++--------------------- fs/wrap.py | 2 +- fs/wrapfs.py | 2 +- fs/zipfs.py | 8 ++--- 23 files changed, 194 insertions(+), 158 deletions(-) diff --git a/fs/base.py b/fs/base.py index 7271a7ca..ecd82a9e 100644 --- a/fs/base.py +++ b/fs/base.py @@ -598,7 +598,7 @@ def readbytes(self, path): def download(self, path, file, chunk_size=None, **options): # type: (Text, BinaryIO, Optional[int], **Any) -> None - """Copies a file from the filesystem to a file-like object. + """Copy a file from the filesystem to a file-like object. This may be more efficient that opening and copying files manually if the filesystem supplies an optimized method. @@ -741,7 +741,7 @@ def getsyspath(self, path): # type: (Text) -> Text """Get the *system path* of a resource. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -777,10 +777,9 @@ def getsyspath(self, path): def getospath(self, path): # type: (Text) -> bytes - """Get a *system path* to a resource, encoded in the operating - system's prefered encoding. + """Get the *system path* to a resource, in the OS' prefered encoding. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -809,7 +808,7 @@ def gettype(self, path): # type: (Text) -> ResourceType """Get the type of a resource. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -847,7 +846,7 @@ def geturl(self, path, purpose="download"): # type: (Text, Text) -> Text """Get the URL to a given resource. - Parameters: + Arguments: path (str): A path on the filesystem purpose (str): A short string that indicates which URL to retrieve for the given path (if there is more than @@ -868,7 +867,7 @@ def hassyspath(self, path): # type: (Text) -> bool """Check if a path maps to a system path. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -886,7 +885,7 @@ def hasurl(self, path, purpose="download"): # type: (Text, Text) -> bool """Check if a path has a corresponding URL. - Parameters: + Arguments: path (str): A path on the filesystem. purpose (str): A purpose parameter, as given in `~fs.base.FS.geturl`. @@ -911,7 +910,7 @@ def isdir(self, path): # type: (Text) -> bool """Check if a path maps to an existing directory. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -930,7 +929,7 @@ def isempty(self, path): A directory is considered empty when it does not contain any file or any directory. - Parameters: + Arguments: path (str): A path to a directory on the filesystem. Returns: @@ -947,7 +946,7 @@ def isfile(self, path): # type: (Text) -> bool """Check if a path maps to an existing file. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -963,7 +962,7 @@ def islink(self, path): # type: (Text) -> bool """Check if a path maps to a symlink. - Parameters: + Arguments: path (str): A path on the filesystem. Returns: @@ -1009,7 +1008,7 @@ def movedir(self, src_path, dst_path, create=False): # type: (Text, Text, bool) -> None """Move directory ``src_path`` to ``dst_path``. - Parameters: + Arguments: src_path (str): Path of source directory on the filesystem. dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created @@ -1443,8 +1442,7 @@ def touch(self, path): def validatepath(self, path): # type: (Text) -> Text - """Check if a path is valid, returning a normalized absolute - path. + """Validate a path, returning a normalized absolute path on sucess. Many filesystems have restrictions on the format of paths they support. This method will check that ``path`` is valid on the diff --git a/fs/compress.py b/fs/compress.py index 2110403b..a3d73033 100644 --- a/fs/compress.py +++ b/fs/compress.py @@ -46,9 +46,9 @@ def write_zip( compression (int): Compression to use (one of the constants defined in the `zipfile` module in the stdlib). Defaults to `zipfile.ZIP_DEFLATED`. - encoding (str): - The encoding to use for filenames. The default is ``"utf-8"``, - use ``"CP437"`` if compatibility with WinZip is desired. + encoding (str): The encoding to use for filenames. The default + is ``"utf-8"``, use ``"CP437"`` if compatibility with WinZip + is desired. walker (~fs.walk.Walker, optional): A `Walker` instance, or `None` to use default walker. You can use this to specify which files you want to compress. @@ -116,6 +116,7 @@ def write_tar( """Write the contents of a filesystem to a tar file. Arguments: + src_fs (~fs.base.FS): The source filesystem to compress. file (str or io.IOBase): Destination file, may be a file name or an open file object. compression (str, optional): Compression to use, or `None` diff --git a/fs/copy.py b/fs/copy.py index 80fcdc6b..6cd34392 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -150,7 +150,7 @@ def copy_file_internal( dst_path, # type: Text ): # type: (...) -> None - """Low level copy, that doesn't call manage_fs or lock. + """Copy a file at low level, without calling `manage_fs` or locking. If the destination exists, and is a file, it will be first truncated. @@ -160,7 +160,7 @@ def copy_file_internal( Arguments: src_fs (FS): Source filesystem. src_path (str): Path to a file on the source filesystem. - dst_fs (FS: Destination filesystem. + dst_fs (FS): Destination filesystem. dst_path (str): Path to a file on the destination filesystem. """ diff --git a/fs/errors.py b/fs/errors.py index 49bc2d6f..25625e28 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -57,7 +57,7 @@ class MissingInfoNamespace(AttributeError): """An expected namespace is missing.""" - def __init__(self, namespace): + def __init__(self, namespace): # noqa: D107 # type: (Text) -> None self.namespace = namespace msg = "namespace '{}' is required for this attribute" @@ -73,7 +73,7 @@ class FSError(Exception): default_message = "Unspecified error" - def __init__(self, msg=None): + def __init__(self, msg=None): # noqa: D107 # type: (Optional[Text]) -> None self._msg = msg or self.default_message super(FSError, self).__init__() @@ -101,7 +101,7 @@ class BulkCopyFailed(FSError): default_message = "One or more copy operations failed (see errors attribute)" - def __init__(self, errors): + def __init__(self, errors): # noqa: D107 self.errors = errors super(BulkCopyFailed, self).__init__() @@ -111,7 +111,7 @@ class CreateFailed(FSError): default_message = "unable to create filesystem, {details}" - def __init__(self, msg=None, exc=None): + def __init__(self, msg=None, exc=None): # noqa: D107 # type: (Optional[Text], Optional[Exception]) -> None self._msg = msg or self.default_message self.details = "" if exc is None else text_type(exc) @@ -139,7 +139,7 @@ class PathError(FSError): default_message = "path '{path}' is invalid" - def __init__(self, path, msg=None): + def __init__(self, path, msg=None): # noqa: D107 # type: (Text, Optional[Text]) -> None self.path = path super(PathError, self).__init__(msg=msg) @@ -159,7 +159,7 @@ class NoURL(PathError): default_message = "path '{path}' has no '{purpose}' URL" - def __init__(self, path, purpose, msg=None): + def __init__(self, path, purpose, msg=None): # noqa: D107 # type: (Text, Text, Optional[Text]) -> None self.purpose = purpose super(NoURL, self).__init__(path, msg=msg) @@ -190,7 +190,7 @@ def __init__( path=None, # type: Optional[Text] exc=None, # type: Optional[Exception] msg=None, # type: Optional[Text] - ): + ): # noqa: D107 # type: (...) -> None self.path = path self.exc = exc @@ -243,7 +243,7 @@ class ResourceError(FSError): default_message = "failed on path {path}" - def __init__(self, path, exc=None, msg=None): + def __init__(self, path, exc=None, msg=None): # noqa: D107 # type: (Text, Optional[Exception], Optional[Text]) -> None self.path = path self.exc = exc @@ -326,7 +326,7 @@ class IllegalBackReference(ValueError): """ - def __init__(self, path): + def __init__(self, path): # noqa: D107 # type: (Text) -> None self.path = path msg = ("path '{path}' contains back-references outside of filesystem").format( diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 0ec93017..99c7b89d 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -347,18 +347,6 @@ def seek(self, pos, whence=Seek.set): class FTPFS(FS): """A FTP (File Transport Protocol) Filesystem. - - Arguments: - host (str): A FTP host, e.g. ``'ftp.mirror.nl'``. - user (str): A username (default is ``'anonymous'``). - passwd (str): Password for the server, or `None` for anon. - acct (str): FTP account. - timeout (int): Timeout for contacting server (in seconds, - defaults to 10). - port (int): FTP port number (default 21). - proxy (str, optional): An FTP proxy, or ``None`` (default) - for no proxy. - """ _meta = { @@ -381,6 +369,20 @@ def __init__( proxy=None, # type: Optional[Text] ): # type: (...) -> None + """Create a new `FTPFS` instance. + + Arguments: + host (str): A FTP host, e.g. ``'ftp.mirror.nl'``. + user (str): A username (default is ``'anonymous'``). + passwd (str): Password for the server, or `None` for anon. + acct (str): FTP account. + timeout (int): Timeout for contacting server (in seconds, + defaults to 10). + port (int): FTP port number (default 21). + proxy (str, optional): An FTP proxy, or ``None`` (default) + for no proxy. + + """ super(FTPFS, self).__init__() self._host = host self._user = user @@ -494,9 +496,9 @@ def _get_ftp(self): return self._ftp @property - def features(self): + def features(self): # noqa: D401 # type: () -> Dict[Text, Text] - """dict: features of the remote FTP server.""" + """`dict`: Features of the remote FTP server.""" self._get_ftp() return self._features diff --git a/fs/glob.py b/fs/glob.py index ffbc75dc..a2c3aaa8 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -1,3 +1,6 @@ +"""Useful functions for working with glob patterns. +""" + from __future__ import unicode_literals from collections import namedtuple @@ -93,18 +96,6 @@ def imatch(pattern, path): class Globber(object): """A generator of glob results. - - Arguments: - fs (~fs.base.FS): A filesystem object - pattern (str): A glob pattern, e.g. ``"**/*.py"`` - path (str): A path to a directory in the filesystem. - namespaces (list): A list of additional info namespaces. - case_sensitive (bool): If ``True``, the path matching will be - case *sensitive* i.e. ``"FOO.py"`` and ``"foo.py"`` will - be different, otherwise path matching will be case *insensitive*. - exclude_dirs (list): A list of patterns to exclude when searching, - e.g. ``["*.git"]``. - """ def __init__( @@ -117,6 +108,20 @@ def __init__( exclude_dirs=None, ): # type: (FS, str, str, Optional[List[str]], bool, Optional[List[str]]) -> None + """Create a new Globber instance. + + Arguments: + fs (~fs.base.FS): A filesystem object + pattern (str): A glob pattern, e.g. ``"**/*.py"`` + path (str): A path to a directory in the filesystem. + namespaces (list): A list of additional info namespaces. + case_sensitive (bool): If ``True``, the path matching will be + case *sensitive* i.e. ``"FOO.py"`` and ``"foo.py"`` will be + different, otherwise path matching will be case *insensitive*. + exclude_dirs (list): A list of patterns to exclude when searching, + e.g. ``["*.git"]``. + + """ self.fs = fs self.pattern = pattern self.path = path @@ -160,7 +165,7 @@ def _make_iter(self, search="breadth", namespaces=None): def __iter__(self): # type: () -> Iterator[GlobMatch] - """An iterator of :class:`fs.glob.GlobMatch` objects.""" + """Get an iterator of :class:`fs.glob.GlobMatch` objects.""" return self._make_iter() def count(self): @@ -200,7 +205,6 @@ def count_lines(self): LineCounts(lines=5767102, non_blank=4915110) """ - lines = 0 non_blank = 0 for path, info in self._make_iter(): @@ -213,7 +217,7 @@ def count_lines(self): def remove(self): # type: () -> int - """Removed all matched paths. + """Remove all matched paths. Returns: int: Number of file and directories removed. @@ -235,13 +239,10 @@ def remove(self): class BoundGlobber(object): - """A :class:`~Globber` object bound to a filesystem. + """A `~fs.glob.Globber` object bound to a filesystem. An instance of this object is available on every Filesystem object - as ``.glob``. - - Arguments: - fs (FS): A filesystem object. + as the `~fs.base.FS.glob` property. """ @@ -249,6 +250,12 @@ class BoundGlobber(object): def __init__(self, fs): # type: (FS) -> None + """Create a new bound Globber. + + Arguments: + fs (FS): A filesystem object to bind to. + + """ self.fs = fs def __repr__(self): @@ -270,9 +277,7 @@ def __call__( e.g. ``["*.git"]``. Returns: - `~Globber`: - An object that may be queried for the glob matches. - + `Globber`: An object that may be queried for the glob matches. """ return Globber( diff --git a/fs/info.py b/fs/info.py index e50801c9..60a659e6 100644 --- a/fs/info.py +++ b/fs/info.py @@ -184,14 +184,20 @@ def name(self): @property def suffix(self): # type: () -> Text - """`str`: the last component of the name (including dot), or an - empty string if there is no suffix. + """`str`: the last component of the name (with dot). + + In case there is no suffix, an empty string is returned. Example: >>> info >>> info.suffix '.py' + >>> info2 + + >>> info2.suffix + '' + """ name = self.get("basic", "name") if name.startswith(".") and name.count(".") == 1: @@ -209,6 +215,7 @@ def suffixes(self): >>> info.suffixes ['.tar', '.gz'] + """ name = self.get("basic", "name") if name.startswith(".") and name.count(".") == 1: diff --git a/fs/iotools.py b/fs/iotools.py index aa1bb92e..bf7f37a5 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -29,7 +29,7 @@ class RawWrapper(io.RawIOBase): """Convert a Python 2 style file-like object in to a IO object.""" - def __init__(self, f, mode=None, name=None): + def __init__(self, f, mode=None, name=None): # noqa: D107 # type: (IO[bytes], Optional[Text], Optional[Text]) -> None self._f = f self.mode = mode or getattr(f, "mode", None) diff --git a/fs/lrucache.py b/fs/lrucache.py index 11858416..02a712e2 100644 --- a/fs/lrucache.py +++ b/fs/lrucache.py @@ -21,6 +21,7 @@ class LRUCache(OrderedDict, typing.Generic[_K, _V]): """ def __init__(self, cache_size): + """Create a new LRUCache with the given size.""" # type: (int) -> None self.cache_size = cache_size super(LRUCache, self).__init__() diff --git a/fs/mirror.py b/fs/mirror.py index ceb8ccd3..cdbf4b01 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -73,6 +73,7 @@ def mirror( workers (int): Number of worker threads used (0 for single threaded). Set to a relatively low number for network filesystems, 4 would be a good start. + """ def src(): diff --git a/fs/mode.py b/fs/mode.py index 35018755..613e1ba9 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -31,12 +31,6 @@ class Mode(typing.Container[Text]): `mode strings `_ used when opening files. - Arguments: - mode (str): A *mode* string, as used by `io.open`. - - Raises: - ValueError: If the mode string is invalid. - Example: >>> mode = Mode('rb') >>> mode.reading @@ -52,6 +46,15 @@ class Mode(typing.Container[Text]): def __init__(self, mode): # type: (Text) -> None + """Create a new `Mode` instance. + + Arguments: + mode (str): A *mode* string, as used by `io.open`. + + Raises: + ValueError: If the mode string is invalid. + + """ self._mode = mode self.validate() diff --git a/fs/mountfs.py b/fs/mountfs.py index e0a4e956..06597404 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -46,11 +46,6 @@ class MountError(Exception): class MountFS(FS): """A virtual filesystem that maps directories on to other file-systems. - - Arguments: - auto_close (bool): If `True` (the default), the child - filesystems will be closed when `MountFS` is closed. - """ _meta = { @@ -62,6 +57,13 @@ class MountFS(FS): } def __init__(self, auto_close=True): + """Create a new `MountFS` instance. + + Arguments: + auto_close (bool): If `True` (the default), the child + filesystems will be closed when `MountFS` is closed. + + """ # type: (bool) -> None super(MountFS, self).__init__() self.auto_close = auto_close diff --git a/fs/move.py b/fs/move.py index 4f6fc2ab..1d8e26c1 100644 --- a/fs/move.py +++ b/fs/move.py @@ -41,7 +41,7 @@ def move_file( Arguments: src_fs (FS or str): Source filesystem (instance or URL). src_path (str): Path to a file on ``src_fs``. - dst_fs (FS or str); Destination filesystem (instance or URL). + dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on ``dst_fs``. """ @@ -72,8 +72,8 @@ def move_dir( src_path (str): Path to a directory on ``src_fs`` dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a directory on ``dst_fs``. - workers (int): Use `worker` threads to copy data, or ``0`` (default) for - a single-threaded copy. + workers (int): Use ``worker`` threads to copy data, or ``0`` + (default) for a single-threaded copy. """ diff --git a/fs/multifs.py b/fs/multifs.py index 3e6abb13..6f0fff42 100644 --- a/fs/multifs.py +++ b/fs/multifs.py @@ -55,6 +55,13 @@ class MultiFS(FS): def __init__(self, auto_close=True): # type: (bool) -> None + """Create a new MultiFS. + + Arguments: + auto_close (bool): If `True` (the default), the child + filesystems will be closed when `MultiFS` is closed. + + """ super(MultiFS, self).__init__() self._auto_close = auto_close diff --git a/fs/opener/registry.py b/fs/opener/registry.py index d01530dc..4c1c2d3e 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -68,6 +68,7 @@ def install(self, opener): @registry.install class ArchiveOpener(Opener): protocols = ['zip', 'tar'] + """ _opener = opener if isinstance(opener, Opener) else opener() assert isinstance(_opener, Opener), "Opener instance required" @@ -80,7 +81,6 @@ class ArchiveOpener(Opener): def protocols(self): # type: () -> List[Text] """`list`: the list of supported protocols.""" - _protocols = list(self._protocols) if self.load_extern: _protocols.extend( diff --git a/fs/permissions.py b/fs/permissions.py index 7e340f89..2efd9113 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -50,19 +50,6 @@ class Permissions(object): on a resource. It supports Linux permissions, but is generic enough to manage permission information from almost any filesystem. - Arguments: - names (list, optional): A list of permissions. - mode (int, optional): A mode integer. - user (str, optional): A triplet of *user* permissions, e.g. - ``"rwx"`` or ``"r--"`` - group (str, optional): A triplet of *group* permissions, e.g. - ``"rwx"`` or ``"r--"`` - other (str, optional): A triplet of *other* permissions, e.g. - ``"rwx"`` or ``"r--"`` - sticky (bool, optional): A boolean for the *sticky* bit. - setuid (bool, optional): A boolean for the *setuid* bit. - setguid (bool, optional): A boolean for the *setguid* bit. - Example: >>> from fs.permissions import Permissions >>> p = Permissions(user='rwx', group='rw-', other='r--') @@ -103,6 +90,22 @@ def __init__( setguid=None, # type: Optional[bool] ): # type: (...) -> None + """Create a new `Permissions` instance. + + Arguments: + names (list, optional): A list of permissions. + mode (int, optional): A mode integer. + user (str, optional): A triplet of *user* permissions, e.g. + ``"rwx"`` or ``"r--"`` + group (str, optional): A triplet of *group* permissions, e.g. + ``"rwx"`` or ``"r--"`` + other (str, optional): A triplet of *other* permissions, e.g. + ``"rwx"`` or ``"r--"`` + sticky (bool, optional): A boolean for the *sticky* bit. + setuid (bool, optional): A boolean for the *setuid* bit. + setguid (bool, optional): A boolean for the *setguid* bit. + + """ if names is not None: self._perms = set(names) elif mode is not None: diff --git a/fs/subfs.py b/fs/subfs.py index 81e19c9d..1357eb1f 100644 --- a/fs/subfs.py +++ b/fs/subfs.py @@ -29,7 +29,7 @@ class SubFS(WrapFS[_F], typing.Generic[_F]): """ - def __init__(self, parent_fs, path): + def __init__(self, parent_fs, path): # noqa: D107 # type: (_F, Text) -> None super(SubFS, self).__init__(parent_fs) self._sub_dir = abspath(normpath(path)) diff --git a/fs/tarfs.py b/fs/tarfs.py index b9b14c0a..85e74840 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -150,7 +150,7 @@ def __init__( compression=None, # type: Optional[Text] encoding="utf-8", # type: Text temp_fs="temp://__tartemp__", # type: Text - ): + ): # noqa: D107 # type: (...) -> None pass @@ -165,7 +165,7 @@ def __init__( compression=None, # type: Optional[Text] encoding="utf-8", # type: Text temp_fs="temp://__tartemp__", # type: Text - ): + ): # noqa: D107 # type: (...) -> None self._file = file # type: Union[Text, BinaryIO] self.compression = compression @@ -221,6 +221,7 @@ def write_tar( Note: This is called automatically when the TarFS is closed. + """ if not self.isclosed(): write_tar( @@ -258,7 +259,7 @@ class ReadTarFS(FS): } @errors.CreateFailed.catch_all - def __init__(self, file, encoding="utf-8"): + def __init__(self, file, encoding="utf-8"): # noqa: D107 # type: (Union[Text, BinaryIO], Text) -> None super(ReadTarFS, self).__init__() self._file = file diff --git a/fs/tempfs.py b/fs/tempfs.py index 33a12e91..32ca828d 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -28,18 +28,6 @@ @six.python_2_unicode_compatible class TempFS(OSFS): """A temporary filesystem on the OS. - - Arguments: - identifier (str): A string to distinguish the directory within - the OS temp location, used as part of the directory name. - temp_dir (str, optional): An OS path to your temp directory - (leave as `None` to auto-detect) - auto_clean (bool): If `True` (the default), the directory - contents will be wiped on close. - ignore_clean_errors (bool): If `True` (the default), any errors - in the clean process will be suppressed. If `False`, they - will be raised. - """ def __init__( @@ -50,6 +38,20 @@ def __init__( ignore_clean_errors=True, # type: bool ): # type: (...) -> None + """Create a new `TempFS` instance. + + Arguments: + identifier (str): A string to distinguish the directory within + the OS temp location, used as part of the directory name. + temp_dir (str, optional): An OS path to your temp directory + (leave as `None` to auto-detect) + auto_clean (bool): If `True` (the default), the directory + contents will be wiped on close. + ignore_clean_errors (bool): If `True` (the default), any errors + in the clean process will be suppressed. If `False`, they + will be raised. + + """ self.identifier = identifier self._auto_clean = auto_clean self._ignore_clean_errors = ignore_clean_errors diff --git a/fs/walk.py b/fs/walk.py index 44a910d1..294cca2c 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -51,32 +51,6 @@ class Walker(object): """A walker object recursively lists directories in a filesystem. - - Arguments: - ignore_errors (bool): If `True`, any errors reading a - directory will be ignored, otherwise exceptions will - be raised. - on_error (callable, optional): If ``ignore_errors`` is `False`, - then this callable will be invoked for a path and the exception - object. It should return `True` to ignore the error, or `False` - to re-raise it. - search (str): If ``'breadth'`` then the directory will be - walked *top down*. Set to ``'depth'`` to walk *bottom up*. - filter (list, optional): If supplied, this parameter should be - a list of filename patterns, e.g. ``['*.py']``. Files will - only be returned if the final component matches one of the - patterns. - exclude (list, optional): If supplied, this parameter should be - a list of filename patterns, e.g. ``['~*']``. Files matching - any of these patterns will be removed from the walk. - filter_dirs (list, optional): A list of patterns that will be used - to match directories paths. The walk will only open directories - that match at least one of these patterns. - exclude_dirs (list, optional): A list of patterns that will be - used to filter out directories from the walk. e.g. - ``['*.svn', '*.git']``. - max_depth (int, optional): Maximum directory depth to walk. - """ def __init__( @@ -91,6 +65,34 @@ def __init__( max_depth=None, # type: Optional[int] ): # type: (...) -> None + """Create a new `Walker` instance. + + Arguments: + ignore_errors (bool): If `True`, any errors reading a + directory will be ignored, otherwise exceptions will + be raised. + on_error (callable, optional): If ``ignore_errors`` is `False`, + then this callable will be invoked for a path and the + exception object. It should return `True` to ignore the error, + or `False` to re-raise it. + search (str): If ``"breadth"`` then the directory will be + walked *top down*. Set to ``"depth"`` to walk *bottom up*. + filter (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``["*.py"]``. Files will + only be returned if the final component matches one of the + patterns. + exclude (list, optional): If supplied, this parameter should be + a list of filename patterns, e.g. ``["~*"]``. Files matching + any of these patterns will be removed from the walk. + filter_dirs (list, optional): A list of patterns that will be used + to match directories paths. The walk will only open directories + that match at least one of these patterns. + exclude_dirs (list, optional): A list of patterns that will be + used to filter out directories from the walk. e.g. + ``['*.svn', '*.git']``. + max_depth (int, optional): Maximum directory depth to walk. + + """ if search not in ("breadth", "depth"): raise ValueError("search must be 'breadth' or 'depth'") self.ignore_errors = ignore_errors @@ -114,20 +116,19 @@ def __init__( @classmethod def _ignore_errors(cls, path, error): # type: (Text, Exception) -> bool - """Default on_error callback.""" + """Ignore dir scan errors when called.""" return True @classmethod def _raise_errors(cls, path, error): # type: (Text, Exception) -> bool - """Callback to re-raise dir scan errors.""" + """Re-raise dir scan errors when called.""" return False @classmethod def _calculate_depth(cls, path): # type: (Text) -> int - """Calculate the 'depth' of a directory path (number of - components). + """Calculate the 'depth' of a directory path (i.e. count components). """ _path = path.strip("/") return _path.count("/") + 1 if _path else 0 @@ -262,7 +263,6 @@ def check_file(self, fs, info): bool: `True` if the file should be included. """ - if self.exclude is not None and fs.match(self.exclude, info.name): return False return fs.match(self.filter, info.name) @@ -492,11 +492,6 @@ def _walk_depth( class BoundWalker(typing.Generic[_F]): """A class that binds a `Walker` instance to a `FS` instance. - Arguments: - fs (FS): A filesystem instance. - walker_class (type): A `~fs.walk.WalkerBase` - sub-class. The default uses `~fs.walk.Walker`. - You will typically not need to create instances of this class explicitly. Filesystems have a `~FS.walk` property which returns a `BoundWalker` object. @@ -507,12 +502,20 @@ class BoundWalker(typing.Generic[_F]): >>> home_fs.walk BoundWalker(OSFS('/Users/will', encoding='utf-8')) - A `BoundWalker` is callable. Calling it is an alias for - `~fs.walk.BoundWalker.walk`. + A `BoundWalker` is callable. Calling it is an alias for the + `~fs.walk.BoundWalker.walk` method. """ def __init__(self, fs, walker_class=Walker): + """Create a new walker bound to the given filesystem. + + Arguments: + fs (FS): A filesystem instance. + walker_class (type): A `~fs.walk.WalkerBase` + sub-class. The default uses `~fs.walk.Walker`. + + """ # type: (_F, Type[Walker]) -> None self.fs = fs self.walker_class = walker_class diff --git a/fs/wrap.py b/fs/wrap.py index 7026bcbc..8685e9f6 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -94,7 +94,7 @@ class WrapCachedDir(WrapFS[_F], typing.Generic[_F]): wrap_name = "cached-dir" - def __init__(self, wrap_fs): + def __init__(self, wrap_fs): # noqa: D107 # type: (_F) -> None super(WrapCachedDir, self).__init__(wrap_fs) self._cache = {} # type: Dict[Tuple[Text, frozenset], Dict[Text, Info]] diff --git a/fs/wrapfs.py b/fs/wrapfs.py index c09e9cf3..e40a7a83 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -60,7 +60,7 @@ class WrapFS(FS, typing.Generic[_F]): wrap_name = None # type: Optional[Text] - def __init__(self, wrap_fs): + def __init__(self, wrap_fs): # noqa: D107 # type: (_F) -> None self._wrap_fs = wrap_fs super(WrapFS, self).__init__() diff --git a/fs/zipfs.py b/fs/zipfs.py index 32d0efcc..d8300a26 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -44,7 +44,7 @@ class _ZipExtFile(RawWrapper): - def __init__(self, fs, name): + def __init__(self, fs, name): # noqa: D107 # type: (ReadZipFS, Text) -> None self._zip = _zip = fs._zip self._end = _zip.getinfo(name).file_size @@ -191,7 +191,7 @@ def __init__( compression=zipfile.ZIP_DEFLATED, # type: int encoding="utf-8", # type: Text temp_fs="temp://__ziptemp__", # type: Text - ): + ): # noqa: D107 # type: (...) -> None pass @@ -206,7 +206,7 @@ def __init__( compression=zipfile.ZIP_DEFLATED, # type: int encoding="utf-8", # type: Text temp_fs="temp://__ziptemp__", # type: Text - ): + ): # noqa: D107 # type: (...) -> None self._file = file self.compression = compression @@ -288,7 +288,7 @@ class ReadZipFS(FS): } @errors.CreateFailed.catch_all - def __init__(self, file, encoding="utf-8"): + def __init__(self, file, encoding="utf-8"): # noqa: D107 # type: (Union[BinaryIO, Text], Text) -> None super(ReadZipFS, self).__init__() self._file = file From 91a57da138cf4898aa6abebfbd9562384ddf8841 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 14:55:22 +0100 Subject: [PATCH 087/309] Refactor `fs.appfs` to avoid copying `__init__` documentation --- docs/source/conf.py | 11 ++++++ fs/appfs.py | 92 ++++++++++++++++----------------------------- 2 files changed, 43 insertions(+), 60 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a5cc0d23..749c3330 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -304,3 +304,14 @@ #texinfo_no_detailmenu = False napoleon_include_special_with_doc = True + + +# -- Options for autodoc ----------------------------------------------------- + +# Configure autodoc so that it doesn't skip building the documentation for +# __init__ methods, since the arguments to instantiate classes should be in +# the __init__ docstring and not at the class-level. + +autodoc_default_options = { + 'special-members': '__init__', +} diff --git a/fs/appfs.py b/fs/appfs.py index 47223e83..d4c0903b 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -9,8 +9,11 @@ # see http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx +import abc import typing +import six + from .osfs import OSFS from ._repr import make_repr from appdirs import AppDirs @@ -29,6 +32,22 @@ ] +class _CopyInitMeta(abc.ABCMeta): + """A metaclass that performs a hard copy of the `__init__`. + + This is a fix for Sphinx, which is a pain to configure in a way that + it documents the ``__init__`` method of a class when it is inherited. + Copying ``__init__`` makes it think it is not inherited, and let us + share the documentation between all the `_AppFS` subclasses. + + """ + + def __new__(mcls, classname, bases, cls_dict): + cls_dict.setdefault("__init__", bases[0].__init__) + return super().__new__(mcls, classname, bases, cls_dict) + + +@six.add_metaclass(_CopyInitMeta) class _AppFS(OSFS): """Abstract base class for an app FS.""" @@ -46,6 +65,19 @@ def __init__( create=True, # type: bool ): # type: (...) -> None + """Create a new application-specific filesystem. + + Arguments: + appname (str): The name of the application. + author (str): The name of the author (used on Windows). + version (str): Optional version string, if a unique location + per version of the application is required. + roaming (bool): If `True`, use a *roaming* profile on + Windows. + create (bool): If `True` (the default) the directory + will be created if it does not exist. + + """ self.app_dirs = AppDirs(appname, author, version, roaming) self._create = create super(_AppFS, self).__init__( @@ -76,16 +108,6 @@ class UserDataFS(_AppFS): May also be opened with ``open_fs('userdata://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "user_data_dir" @@ -97,16 +119,6 @@ class UserConfigFS(_AppFS): May also be opened with ``open_fs('userconf://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "user_config_dir" @@ -118,16 +130,6 @@ class UserCacheFS(_AppFS): May also be opened with ``open_fs('usercache://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "user_cache_dir" @@ -139,16 +141,6 @@ class SiteDataFS(_AppFS): May also be opened with ``open_fs('sitedata://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "site_data_dir" @@ -160,16 +152,6 @@ class SiteConfigFS(_AppFS): May also be opened with ``open_fs('siteconf://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "site_config_dir" @@ -181,16 +163,6 @@ class UserLogFS(_AppFS): May also be opened with ``open_fs('userlog://appname:author:version')``. - Arguments: - appname (str): The name of the application. - author (str): The name of the author (used on Windows). - version (str): Optional version string, if a unique location - per version of the application is required. - roaming (bool): If `True`, use a *roaming* profile on - Windows. - create (bool): If `True` (the default) the directory - will be created if it does not exist. - """ app_dir = "user_log_dir" From 63a4cd113feda72fa8e2de1dd65d8dba26949514 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 14:59:26 +0100 Subject: [PATCH 088/309] Move `OSFS` initialization parameters to the `OSFS.__init__` docstring --- fs/osfs.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index ed091e64..3b35541c 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -81,22 +81,6 @@ class OSFS(FS): """Create an OSFS. - Arguments: - root_path (str or ~os.PathLike): An OS path or path-like object to - the location on your HD you wish to manage. - create (bool): Set to `True` to create the root directory if it - does not already exist, otherwise the directory should exist - prior to creating the ``OSFS`` instance (defaults to `False`). - create_mode (int): The permissions that will be used to create - the directory if ``create`` is `True` and the path doesn't - exist, defaults to ``0o777``. - expand_vars(bool): If `True` (the default) environment variables of - the form $name or ${name} will be expanded. - - Raises: - `fs.errors.CreateFailed`: If ``root_path`` does not - exist, or could not be created. - Examples: >>> current_directory_fs = OSFS('.') >>> home_fs = OSFS('~/') @@ -112,7 +96,25 @@ def __init__( expand_vars=True, # type: bool ): # type: (...) -> None - """Create an OSFS instance.""" + """Create an OSFS instance. + + Arguments: + root_path (str or ~os.PathLike): An OS path or path-like object + to the location on your HD you wish to manage. + create (bool): Set to `True` to create the root directory if it + does not already exist, otherwise the directory should exist + prior to creating the ``OSFS`` instance (defaults to `False`). + create_mode (int): The permissions that will be used to create + the directory if ``create`` is `True` and the path doesn't + exist, defaults to ``0o777``. + expand_vars(bool): If `True` (the default) environment variables + of the form ``~``, ``$name`` or ``${name}`` will be expanded. + + Raises: + `fs.errors.CreateFailed`: If ``root_path`` does not + exist, or could not be created. + + """ super(OSFS, self).__init__() if isinstance(root_path, bytes): root_path = fsdecode(root_path) From b19862b1904adb2e13bc608982c2970da7ac802d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 14:59:42 +0100 Subject: [PATCH 089/309] Fix rendering of examples in `MemoryFS` docstring --- fs/memoryfs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 3164d721..30e9c21f 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -304,12 +304,16 @@ class MemoryFS(FS): fast, but non-permanent. The `MemoryFS` constructor takes no arguments. - Example: - >>> mem_fs = MemoryFS() + Examples: + Create with the constructor:: - Or via an FS URL: - >>> import fs - >>> mem_fs = fs.open_fs('mem://') + >>> from fs.memoryfs import MemoryFS + >>> mem_fs = MemoryFS() + + Or via an FS URL:: + + >>> import fs + >>> mem_fs = fs.open_fs('mem://') """ From 66d49a8b156514b46f65631b7b389ee1e1af2000 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 15:07:05 +0100 Subject: [PATCH 090/309] Rename linter environments in `tox.ini` to be tool-agnostic --- .github/workflows/test.yml | 8 ++++---- tox.ini | 17 ++++++++++------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9647b2c..19994538 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,10 +50,10 @@ jobs: fail-fast: false matrix: linter: - - mypy - - flake8 - - black - - pydocstyle + - typecheck + - codestyle + - docstyle + - codeformat steps: - name: Checkout code uses: actions/checkout@v1 diff --git a/tox.ini b/tox.ini index 23ce73fc..d71aa781 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,9 @@ [tox] -envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, mypy, lint +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, typecheck, codestyle, docstyle, codeformat sitepackages = false skip_missing_interpreters = true +requires = + setuptools >=38.3.0 [testenv] commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests @@ -16,14 +18,14 @@ deps = scandir: .[scandir] !scandir: . -[testenv:mypy] +[testenv:typecheck] commands = mypy --config-file {toxinidir}/setup.cfg {toxinidir}/fs deps = . mypy==0.800 -[testenv:flake8] -commands = flake8 {toxinidir}/fs {toxinidir}/tests +[testenv:codestyle] +commands = flake8 --config={toxinidir}/setup.cfg {toxinidir}/fs {toxinidir}/tests deps = flake8==3.7.9 #flake8-builtins==1.5.3 @@ -32,14 +34,15 @@ deps = flake8-mutable==1.2.0 flake8-tuple==0.4.0 -[testenv:black] +[testenv:codeformat] commands = black --check {toxinidir}/fs deps = black==20.8b1 -[testenv:pydocstyle] -python = python3.9 +[testenv:docstyle] commands = pydocstyle --config={toxinidir}/setup.cfg {toxinidir}/fs +deps = + pydocstyle==5.1.1 [gh-actions] python = From 5d8009cebae52285479982a8766666cbd64d013e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 15:08:48 +0100 Subject: [PATCH 091/309] Fix remaining linter issues in `fs` modules --- fs/appfs.py | 2 +- fs/ftpfs.py | 3 +-- fs/glob.py | 3 +-- fs/lrucache.py | 2 +- fs/mirror.py | 2 +- fs/mountfs.py | 5 ++--- fs/permissions.py | 2 +- fs/tempfs.py | 3 +-- fs/walk.py | 8 +++----- 9 files changed, 12 insertions(+), 18 deletions(-) diff --git a/fs/appfs.py b/fs/appfs.py index d4c0903b..131ea8a8 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -44,7 +44,7 @@ class _CopyInitMeta(abc.ABCMeta): def __new__(mcls, classname, bases, cls_dict): cls_dict.setdefault("__init__", bases[0].__init__) - return super().__new__(mcls, classname, bases, cls_dict) + return super(abc.ABCMeta, mcls).__new__(mcls, classname, bases, cls_dict) @six.add_metaclass(_CopyInitMeta) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 99c7b89d..d2e37d7f 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -346,8 +346,7 @@ def seek(self, pos, whence=Seek.set): class FTPFS(FS): - """A FTP (File Transport Protocol) Filesystem. - """ + """A FTP (File Transport Protocol) Filesystem.""" _meta = { "invalid_path_chars": "\0", diff --git a/fs/glob.py b/fs/glob.py index a2c3aaa8..c21bb9c6 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -95,8 +95,7 @@ def imatch(pattern, path): class Globber(object): - """A generator of glob results. - """ + """A generator of glob results.""" def __init__( self, diff --git a/fs/lrucache.py b/fs/lrucache.py index 02a712e2..9deb98bd 100644 --- a/fs/lrucache.py +++ b/fs/lrucache.py @@ -21,8 +21,8 @@ class LRUCache(OrderedDict, typing.Generic[_K, _V]): """ def __init__(self, cache_size): - """Create a new LRUCache with the given size.""" # type: (int) -> None + """Create a new LRUCache with the given size.""" self.cache_size = cache_size super(LRUCache, self).__init__() diff --git a/fs/mirror.py b/fs/mirror.py index cdbf4b01..6b989e63 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -73,7 +73,7 @@ def mirror( workers (int): Number of worker threads used (0 for single threaded). Set to a relatively low number for network filesystems, 4 would be a good start. - + """ def src(): diff --git a/fs/mountfs.py b/fs/mountfs.py index 06597404..5e590637 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -45,8 +45,7 @@ class MountError(Exception): class MountFS(FS): - """A virtual filesystem that maps directories on to other file-systems. - """ + """A virtual filesystem that maps directories on to other file-systems.""" _meta = { "virtual": True, @@ -57,6 +56,7 @@ class MountFS(FS): } def __init__(self, auto_close=True): + # type: (bool) -> None """Create a new `MountFS` instance. Arguments: @@ -64,7 +64,6 @@ def __init__(self, auto_close=True): filesystems will be closed when `MountFS` is closed. """ - # type: (bool) -> None super(MountFS, self).__init__() self.auto_close = auto_close self.default_fs = MemoryFS() # type: FS diff --git a/fs/permissions.py b/fs/permissions.py index 2efd9113..032c3be0 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -104,7 +104,7 @@ def __init__( sticky (bool, optional): A boolean for the *sticky* bit. setuid (bool, optional): A boolean for the *setuid* bit. setguid (bool, optional): A boolean for the *setguid* bit. - + """ if names is not None: self._perms = set(names) diff --git a/fs/tempfs.py b/fs/tempfs.py index 32ca828d..a1e5a3d2 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -27,8 +27,7 @@ @six.python_2_unicode_compatible class TempFS(OSFS): - """A temporary filesystem on the OS. - """ + """A temporary filesystem on the OS.""" def __init__( self, diff --git a/fs/walk.py b/fs/walk.py index 294cca2c..0f4adb99 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -50,8 +50,7 @@ class Walker(object): - """A walker object recursively lists directories in a filesystem. - """ + """A walker object recursively lists directories in a filesystem.""" def __init__( self, @@ -128,8 +127,7 @@ def _raise_errors(cls, path, error): @classmethod def _calculate_depth(cls, path): # type: (Text) -> int - """Calculate the 'depth' of a directory path (i.e. count components). - """ + """Calculate the 'depth' of a directory path (i.e. count components).""" _path = path.strip("/") return _path.count("/") + 1 if _path else 0 @@ -508,6 +506,7 @@ class BoundWalker(typing.Generic[_F]): """ def __init__(self, fs, walker_class=Walker): + # type: (_F, Type[Walker]) -> None """Create a new walker bound to the given filesystem. Arguments: @@ -516,7 +515,6 @@ def __init__(self, fs, walker_class=Walker): sub-class. The default uses `~fs.walk.Walker`. """ - # type: (_F, Type[Walker]) -> None self.fs = fs self.walker_class = walker_class From 79fe038b2802dfb7989498fd034205739fa7b756 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 15:32:36 +0100 Subject: [PATCH 092/309] Mark BZ2 and XZ tests as slow in `tests.test_tarfs` --- tests/mark.py | 2 ++ tests/test_ftpfs.py | 11 ++++------- tests/test_tarfs.py | 7 +++++++ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 tests/mark.py diff --git a/tests/mark.py b/tests/mark.py new file mode 100644 index 00000000..4ac89d59 --- /dev/null +++ b/tests/mark.py @@ -0,0 +1,2 @@ +def slow(self): + pass diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 3709dc59..632ec11b 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -27,12 +27,9 @@ from fs.test import FSTestCases try: - from pytest.mark import slow + from pytest import mark except ImportError: - - def slow(cls): - return cls - + from . import mark # Prevent socket timeouts from slowing tests too much socket.setdefaulttimeout(1) @@ -135,7 +132,7 @@ def test_manager_with_host(self): ) -@slow +@mark.slow class TestFTPFS(FSTestCases, unittest.TestCase): user = "user" @@ -285,7 +282,7 @@ def test_features(self): pass -@slow +@mark.slow class TestAnonFTPFS(FSTestCases, unittest.TestCase): user = "anonymous" diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index 2a03afec..fc3f0779 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -18,6 +18,11 @@ from .test_archives import ArchiveTestCases +try: + from pytest import mark +except ImportError: + from . import mark + class TestWriteReadTarFS(unittest.TestCase): def setUp(self): @@ -93,6 +98,7 @@ def destroy_fs(self, fs): del fs._tar_file +@mark.slow @unittest.skipIf(six.PY2, "Python2 does not support LZMA") class TestWriteXZippedTarFS(FSTestCases, unittest.TestCase): def make_fs(self): @@ -118,6 +124,7 @@ def assert_is_xz(self, fs): tarfile.open(fs._tar_file, "r:{}".format(other_comps)) +@mark.slow class TestWriteBZippedTarFS(FSTestCases, unittest.TestCase): def make_fs(self): fh, _tar_file = tempfile.mkstemp() From 9ab0bd598e67d718457229709fc78facd66d67f0 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 15:33:20 +0100 Subject: [PATCH 093/309] Make upload /download tests in `FSTestCases` faster by only building data once --- fs/test.py | 46 +++++++++++++++++++++------------------------ tests/test_appfs.py | 15 ++++++++------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/fs/test.py b/fs/test.py index 4c630137..d8b5b812 100644 --- a/fs/test.py +++ b/fs/test.py @@ -247,6 +247,11 @@ class FSTestCases(object): """Basic FS tests.""" + data1 = b"foo" * 256 * 1024 + data2 = b"bar" * 2 * 256 * 1024 + data3 = b"baz" * 3 * 256 * 1024 + data4 = b"egg" * 7 * 256 * 1024 + def make_fs(self): """Return an FS instance.""" raise NotImplementedError("implement me") @@ -1191,22 +1196,17 @@ def test_copy(self): def _test_upload(self, workers): """Test fs.copy with varying number of worker threads.""" - data1 = b"foo" * 256 * 1024 - data2 = b"bar" * 2 * 256 * 1024 - data3 = b"baz" * 3 * 256 * 1024 - data4 = b"egg" * 7 * 256 * 1024 - with open_fs("temp://") as src_fs: - src_fs.writebytes("foo", data1) - src_fs.writebytes("bar", data2) - src_fs.makedir("dir1").writebytes("baz", data3) - src_fs.makedirs("dir2/dir3").writebytes("egg", data4) + src_fs.writebytes("foo", self.data1) + src_fs.writebytes("bar", self.data2) + src_fs.makedir("dir1").writebytes("baz", self.data3) + src_fs.makedirs("dir2/dir3").writebytes("egg", self.data4) dst_fs = self.fs fs.copy.copy_fs(src_fs, dst_fs, workers=workers) - self.assertEqual(dst_fs.readbytes("foo"), data1) - self.assertEqual(dst_fs.readbytes("bar"), data2) - self.assertEqual(dst_fs.readbytes("dir1/baz"), data3) - self.assertEqual(dst_fs.readbytes("dir2/dir3/egg"), data4) + self.assertEqual(dst_fs.readbytes("foo"), self.data1) + self.assertEqual(dst_fs.readbytes("bar"), self.data2) + self.assertEqual(dst_fs.readbytes("dir1/baz"), self.data3) + self.assertEqual(dst_fs.readbytes("dir2/dir3/egg"), self.data4) def test_upload_0(self): self._test_upload(0) @@ -1222,21 +1222,17 @@ def test_upload_4(self): def _test_download(self, workers): """Test fs.copy with varying number of worker threads.""" - data1 = b"foo" * 256 * 1024 - data2 = b"bar" * 2 * 256 * 1024 - data3 = b"baz" * 3 * 256 * 1024 - data4 = b"egg" * 7 * 256 * 1024 src_fs = self.fs with open_fs("temp://") as dst_fs: - src_fs.writebytes("foo", data1) - src_fs.writebytes("bar", data2) - src_fs.makedir("dir1").writebytes("baz", data3) - src_fs.makedirs("dir2/dir3").writebytes("egg", data4) + src_fs.writebytes("foo", self.data1) + src_fs.writebytes("bar", self.data2) + src_fs.makedir("dir1").writebytes("baz", self.data3) + src_fs.makedirs("dir2/dir3").writebytes("egg", self.data4) fs.copy.copy_fs(src_fs, dst_fs, workers=workers) - self.assertEqual(dst_fs.readbytes("foo"), data1) - self.assertEqual(dst_fs.readbytes("bar"), data2) - self.assertEqual(dst_fs.readbytes("dir1/baz"), data3) - self.assertEqual(dst_fs.readbytes("dir2/dir3/egg"), data4) + self.assertEqual(dst_fs.readbytes("foo"), self.data1) + self.assertEqual(dst_fs.readbytes("bar"), self.data2) + self.assertEqual(dst_fs.readbytes("dir1/baz"), self.data3) + self.assertEqual(dst_fs.readbytes("dir2/dir3/egg"), self.data4) def test_download_0(self): self._test_download(0) diff --git a/tests/test_appfs.py b/tests/test_appfs.py index 9bd52be6..acc8a7f7 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -15,12 +15,13 @@ from fs import appfs -class _TestAppFS(object): +class _TestAppFS(fs.test.FSTestCases): AppFS = None @classmethod def setUpClass(cls): + super(_TestAppFS, cls).setUpClass() cls.tmpdir = tempfile.mkdtemp() @classmethod @@ -62,25 +63,25 @@ def test_str(self): ) -class TestUserDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestUserDataFS(_TestAppFS, unittest.TestCase): AppFS = appfs.UserDataFS -class TestUserConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestUserConfigFS(_TestAppFS, unittest.TestCase): AppFS = appfs.UserConfigFS -class TestUserCacheFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestUserCacheFS(_TestAppFS, unittest.TestCase): AppFS = appfs.UserCacheFS -class TestSiteDataFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestSiteDataFS(_TestAppFS, unittest.TestCase): AppFS = appfs.SiteDataFS -class TestSiteConfigFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestSiteConfigFS(_TestAppFS, unittest.TestCase): AppFS = appfs.SiteConfigFS -class TestUserLogFS(_TestAppFS, fs.test.FSTestCases, unittest.TestCase): +class TestUserLogFS(_TestAppFS, unittest.TestCase): AppFS = appfs.UserLogFS From 2b1cab4aec2a79bd893cb15055dd5694c3e5db1a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 15:51:15 +0100 Subject: [PATCH 094/309] Run PyPy tests in Actions CI but skip some `tests.test_ftpfs` test cases --- .github/workflows/test.yml | 3 +++ tests/test_ftpfs.py | 4 +++- tox.ini | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 19994538..52516c34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,9 @@ jobs: - 3.7 - 3.8 - 3.9 + - pypy-2.7 + - pypy-3.6 + - pypy-3.7 steps: - name: Checkout code uses: actions/checkout@v1 diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 632ec11b..fd34c134 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -3,10 +3,10 @@ from __future__ import print_function from __future__ import unicode_literals -import socket import os import platform import shutil +import socket import tempfile import time import unittest @@ -133,6 +133,7 @@ def test_manager_with_host(self): @mark.slow +@unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestFTPFS(FSTestCases, unittest.TestCase): user = "user" @@ -283,6 +284,7 @@ def test_features(self): @mark.slow +@unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestAnonFTPFS(FSTestCases, unittest.TestCase): user = "anonymous" diff --git a/tox.ini b/tox.ini index d71aa781..04234a0b 100644 --- a/tox.ini +++ b/tox.ini @@ -53,3 +53,6 @@ python = 3.7: py37 3.8: py38 3.9: py39 + pypy-2.7: pypy27 + pypy-3.6: pypy36 + pypy-3.7: pypy37 From 996d20256b47e4f304b5af029e5a3c9b6eea89ba Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 16:33:35 +0100 Subject: [PATCH 095/309] Update all badges on `README.md` to use `shields.io` [ci skip] --- README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 787b29ec..0c4fe2ba 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,14 @@ Python's Filesystem abstraction layer. -[![PyPI version](https://badge.fury.io/py/fs.svg)](https://badge.fury.io/py/fs) +[![PyPI version](https://img.shields.io/pypi/v/fs)](https://pypi.org/project/fs/) [![PyPI](https://img.shields.io/pypi/pyversions/fs.svg)](https://pypi.org/project/fs/) -[![Downloads](https://pepy.tech/badge/fs/month)](https://pepy.tech/project/fs/month) - - -[![Build Status](https://travis-ci.org/PyFilesystem/pyfilesystem2.svg?branch=master)](https://travis-ci.org/PyFilesystem/pyfilesystem2) -[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/pyfilesystem/pyfilesystem2?branch=master&svg=true)](https://ci.appveyor.com/project/willmcgugan/pyfilesystem2) -[![Coverage Status](https://coveralls.io/repos/github/PyFilesystem/pyfilesystem2/badge.svg)](https://coveralls.io/github/PyFilesystem/pyfilesystem2) -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/30ad6445427349218425d93886ade9ee)](https://www.codacy.com/app/will-mcgugan/pyfilesystem2?utm_source=github.com&utm_medium=referral&utm_content=PyFilesystem/pyfilesystem2&utm_campaign=Badge_Grade) +[![Downloads](https://pepy.tech/badge/fs/month)](https://pepy.tech/project/fs/) +[![Build Status](https://img.shields.io/github/workflow/status/PyFilesystem/pyfilesystem2/Test/master?logo=github&cacheSeconds=600)](https://github.com/PyFilesystem/pyfilesystem2/actions?query=branch%3Amaster) +[![Windows Build Status](https://img.shields.io/appveyor/build/willmcgugan/pyfilesystem2/master?logo=appveyor&cacheSeconds=600)](https://ci.appveyor.com/project/willmcgugan/pyfilesystem2) +[![Coverage Status](https://img.shields.io/coveralls/github/PyFilesystem/pyfilesystem2/master?cacheSeconds=600)](https://coveralls.io/github/PyFilesystem/pyfilesystem2) +[![Codacy Badge](https://img.shields.io/codacy/grade/30ad6445427349218425d93886ade9ee/master?logo=codacy)](https://www.codacy.com/app/will-mcgugan/pyfilesystem2?utm_source=github.com&utm_medium=referral&utm_content=PyFilesystem/pyfilesystem2&utm_campaign=Badge_Grade) +[![Docs](https://img.shields.io/readthedocs/pyfilesystem2?maxAge=3600)](http://pyfilesystem2.readthedocs.io/en/stable/?badge=stable) ## Documentation From 2673eed5c044ca86a6f0c2bf93e6cb7927827f48 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 16:50:03 +0100 Subject: [PATCH 096/309] Move `tox` configuration into `setup.cfg` to reduce project repo cluttering --- setup.cfg | 99 ++++++++++++++++++++++++++++++++++++++++++++++--------- tox.ini | 58 -------------------------------- 2 files changed, 83 insertions(+), 74 deletions(-) delete mode 100644 tox.ini diff --git a/setup.cfg b/setup.cfg index 109173e0..faad103c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ -[bdist_wheel] -universal = 1 +# --- Project configuration ------------------------------------------------- [metadata] version = attr: fs._version.__version__ @@ -60,6 +59,11 @@ exclude = tests [options.package_data] fs = py.typed +[bdist_wheel] +universal = 1 + +# --- Individual linter configuration --------------------------------------- + [pydocstyle] inherit = false ignore = D102,D105,D200,D203,D213,D406,D407 @@ -83,6 +87,24 @@ warn_return_any = false [mypy-fs.test] disallow_untyped_defs = false +[flake8] +extend-ignore = E203,E402,W503 +max-line-length = 88 +per-file-ignores = + fs/__init__.py:F401 + fs/*/__init__.py:F401 + tests/*:E501 + fs/opener/*:F811 + fs/_fscompat.py:F401 + +[isort] +default_section = THIRD_PARTY +known_first_party = fs +known_standard_library = typing +line_length = 88 + +# --- Test and coverage configuration ------------------------------------------ + [coverage:run] branch = true omit = fs/test.py @@ -102,18 +124,63 @@ exclude_lines = markers = slow: marks tests as slow (deselect with '-m "not slow"') -[flake8] -extend-ignore = E203,E402,W503 -max-line-length = 88 -per-file-ignores = - fs/__init__.py:F401 - fs/*/__init__.py:F401 - tests/*:E501 - fs/opener/*:F811 - fs/_fscompat.py:F401 +# --- Tox automation configuration --------------------------------------------- -[isort] -default_section = THIRD_PARTY -known_first_party = fs -known_standard_library = typing -line_length = 88 +[tox:tox] +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, typecheck, codestyle, docstyle, codeformat +sitepackages = false +skip_missing_interpreters = true +requires = + setuptools >=38.3.0 + +[testenv] +commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests +deps = + -rtests/requirements.txt + pytest-cov~=2.11 + coverage~=5.0 + py{36,37,38,39}: pytest~=6.2 + py{27,34}: pytest~=4.6 + py{36,37,38,39}: pytest-randomly~=3.5 + py{27,34}: pytest-randomly~=1.2 + scandir: .[scandir] + !scandir: . + +[testenv:typecheck] +commands = mypy --config-file {toxinidir}/setup.cfg {toxinidir}/fs +deps = + . + mypy==0.800 + +[testenv:codestyle] +commands = flake8 --config={toxinidir}/setup.cfg {toxinidir}/fs {toxinidir}/tests +deps = + flake8==3.7.9 + #flake8-builtins==1.5.3 + flake8-bugbear==19.8.0 + flake8-comprehensions==3.1.4 + flake8-mutable==1.2.0 + flake8-tuple==0.4.0 + +[testenv:codeformat] +commands = black --check {toxinidir}/fs +deps = + black==20.8b1 + +[testenv:docstyle] +commands = pydocstyle --config={toxinidir}/setup.cfg {toxinidir}/fs +deps = + pydocstyle==5.1.1 + +[gh-actions] +python = + 2.7: py27, py27-scandir + 3.4: py34, py34-scandir + 3.5: py35 + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 + pypy-2.7: pypy27 + pypy-3.6: pypy36 + pypy-3.7: pypy37 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 04234a0b..00000000 --- a/tox.ini +++ /dev/null @@ -1,58 +0,0 @@ -[tox] -envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, typecheck, codestyle, docstyle, codeformat -sitepackages = false -skip_missing_interpreters = true -requires = - setuptools >=38.3.0 - -[testenv] -commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests -deps = - -rtests/requirements.txt - pytest-cov~=2.11 - coverage~=5.0 - py{36,37,38,39}: pytest~=6.2 - py{27,34}: pytest~=4.6 - py{36,37,38,39}: pytest-randomly~=3.5 - py{27,34}: pytest-randomly~=1.2 - scandir: .[scandir] - !scandir: . - -[testenv:typecheck] -commands = mypy --config-file {toxinidir}/setup.cfg {toxinidir}/fs -deps = - . - mypy==0.800 - -[testenv:codestyle] -commands = flake8 --config={toxinidir}/setup.cfg {toxinidir}/fs {toxinidir}/tests -deps = - flake8==3.7.9 - #flake8-builtins==1.5.3 - flake8-bugbear==19.8.0 - flake8-comprehensions==3.1.4 - flake8-mutable==1.2.0 - flake8-tuple==0.4.0 - -[testenv:codeformat] -commands = black --check {toxinidir}/fs -deps = - black==20.8b1 - -[testenv:docstyle] -commands = pydocstyle --config={toxinidir}/setup.cfg {toxinidir}/fs -deps = - pydocstyle==5.1.1 - -[gh-actions] -python = - 2.7: py27, py27-scandir - 3.4: py34, py34-scandir - 3.5: py35 - 3.6: py36 - 3.7: py37 - 3.8: py38 - 3.9: py39 - pypy-2.7: pypy27 - pypy-3.6: pypy36 - pypy-3.7: pypy37 From e95713941606197688c16a91436a29bbaf6612fb Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 16:51:30 +0100 Subject: [PATCH 097/309] Remove Travis-CI configuration files --- .travis.yml | 57 -------------------------------------------- testrequirements.txt | 11 --------- 2 files changed, 68 deletions(-) delete mode 100644 .travis.yml delete mode 100644 testrequirements.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 89b1cce9..00000000 --- a/.travis.yml +++ /dev/null @@ -1,57 +0,0 @@ -dist: xenial -sudo: false -language: python - -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" - - "3.7" - - "3.8" - - "3.9" - - "pypy" - - "pypy3.5-7.0" # Need 7.0+ due to a bug in earlier versions that broke our tests. - -matrix: - include: - - name: "Type checking" - python: "3.7" - env: TOXENV=typecheck - - name: "Lint" - python: "3.7" - env: TOXENV=lint - - # Temporary bandaid for https://github.com/PyFilesystem/pyfilesystem2/issues/342 - allow_failures: - - python: pypy - - python: pypy3.5-7.0 - -before_install: - - pip install -U tox tox-travis - - pip --version - - pip install -r testrequirements.txt - - pip freeze - -install: - - pip install -e . - -# command to run tests -script: tox - -after_success: - - coveralls - -before_deploy: - - pip install -U twine wheel - - python setup.py sdist bdist_wheel - -deploy: - provider: script - script: twine upload dist/* - skip_cleanup: true - on: - python: 3.9 - tags: true - repo: PyFilesystem/pyfilesystem2 - diff --git a/testrequirements.txt b/testrequirements.txt deleted file mode 100644 index bfa0e294..00000000 --- a/testrequirements.txt +++ /dev/null @@ -1,11 +0,0 @@ -pytest==4.6.5 -pytest-cov==2.7.1 -pytest-randomly==1.2.3 ; python_version<"3.5" -pytest-randomly==3.0.0 ; python_version>="3.5" -mock==3.0.5 ; python_version<"3.3" -pyftpdlib==1.5.5 - -# Not directly required. `pyftpdlib` appears to need these but doesn't list them -# as requirements. -psutil -pysendfile From 3624d47c5a8dc2149fe542f9203c391184154d2a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 28 Jan 2021 17:10:22 +0100 Subject: [PATCH 098/309] Update `CHANGELOG.md` with GitHub Actions PR and update older entries --- CHANGELOG.md | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48676340..b75569fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,22 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased ### Changed -- Make `FS.upload` explicit about the expected error when the parent directory of the destination does not exist - [#445](https://github.com/PyFilesystem/pyfilesystem2/pull/445). + +- Make `FS.upload` explicit about the expected error when the parent directory of the destination does not exist. + Closes [#445](https://github.com/PyFilesystem/pyfilesystem2/pull/445). +- Migrate continuous integration from Travis-CI to GitHub Actions and introduce several linters + again in the build steps ([#448](https://github.com/PyFilesystem/pyfilesystem2/pull/448)). + Closes [#446](https://github.com/PyFilesystem/pyfilesystem2/pull/446). +- Stop requiring `pytest` to run tests, allowing any test runner supporting `unittest`-style + test suites. +- `FSTestCases` now builds the large data required for `upload` and `download` tests only + once in order to reduce the total testing time. + +### Fixed + +- Make `FTPFile`, `MemoryFile` and `RawWrapper` accept [`array.array`](https://docs.python.org/3/library/array.html) + arguments for the `write` and `writelines` methods, as expected by their base class [`io.RawIOBase`](https://docs.python.org/3/library/io.html#io.RawIOBase). +- Various documentation issues, including `MemoryFS` docstring not rendering properly. ## [2.4.12] - 2021-01-14 @@ -22,6 +36,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). [#380](https://github.com/PyFilesystem/pyfilesystem2/issues/380). - Added compatibility if a Windows FTP server returns file information to the `LIST` command with 24-hour times. Closes [#438](https://github.com/PyFilesystem/pyfilesystem2/issues/438). +- Added Python 3.9 support. Closes [#443](https://github.com/PyFilesystem/pyfilesystem2/issues/443). ### Changed @@ -30,25 +45,30 @@ and this project adheres to [Semantic Versioning](http://semver.org/). be able to see if we break something aside from known issues with FTP tests. - Include docs in source distributions as well as the whole tests folder, ensuring `conftest.py` is present, fixes [#364](https://github.com/PyFilesystem/pyfilesystem2/issues/364). -- Stop patching copy with Python 3.8+ because it already uses `sendfile`. +- Stop patching copy with Python 3.8+ because it already uses `sendfile` + ([#424](https://github.com/PyFilesystem/pyfilesystem2/pull/424)). + Closes [#421](https://github.com/PyFilesystem/pyfilesystem2/issues/421). ### Fixed - Fixed crash when CPython's -OO flag is used -- Fixed error when parsing timestamps from a FTP directory served from a WindowsNT FTP Server, fixes [#395](https://github.com/PyFilesystem/pyfilesystem2/issues/395). +- Fixed error when parsing timestamps from a FTP directory served from a WindowsNT FTP Server. + Closes [#395](https://github.com/PyFilesystem/pyfilesystem2/issues/395). - Fixed documentation of `Mode.to_platform_bin`. Closes [#382](https://github.com/PyFilesystem/pyfilesystem2/issues/382). - Fixed the code example in the "Testing Filesystems" section of the "Implementing Filesystems" guide. Closes [#407](https://github.com/PyFilesystem/pyfilesystem2/issues/407). - Fixed `FTPFS.openbin` not implicitly opening files in binary mode like expected from `openbin`. Closes [#406](https://github.com/PyFilesystem/pyfilesystem2/issues/406). + ## [2.4.11] - 2019-09-07 ### Added - Added geturl for TarFS and ZipFS for 'fs' purpose. NoURL for 'download' purpose. -- Added helpful root path in CreateFailed exception [#340](https://github.com/PyFilesystem/pyfilesystem2/issues/340) -- Added Python 3.8 support +- Added helpful root path in CreateFailed exception. + Closes [#340](https://github.com/PyFilesystem/pyfilesystem2/issues/340). +- Added Python 3.8 support. ### Fixed @@ -76,7 +96,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed -- Fixed broken WrapFS.movedir [#322](https://github.com/PyFilesystem/pyfilesystem2/issues/322) +- Fixed broken WrapFS.movedir [#322](https://github.com/PyFilesystem/pyfilesystem2/issues/322). ## [2.4.9] - 2019-07-28 @@ -458,7 +478,7 @@ No changes, pushed wrong branch to PyPi. ### Added -- New `copy_if_newer' functionality in`copy` module. +- New `copy_if_newer` functionality in `copy` module. ### Fixed @@ -469,17 +489,17 @@ No changes, pushed wrong branch to PyPi. ### Changed - Improved FTP support for non-compliant servers -- Fix for ZipFS implied directories +- Fix for `ZipFS` implied directories ## [2.0.1] - 2017-03-11 ### Added -- TarFS contributed by Martin Larralde +- `TarFS` contributed by Martin Larralde. ### Fixed -- FTPFS bugs. +- `FTPFS` bugs. ## [2.0.0] - 2016-12-07 From 515ca26b32966174846584d9e315dfdbd784ce83 Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 17:16:27 +0000 Subject: [PATCH 099/309] FTPFS: Add FTPS support. Try to import FTP_TLS while still supporting older Python versions, add a 'tls' option to the FTPFS class, and show the 'ftps://' scheme in the repr when enabled. --- fs/ftpfs.py | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index d2e37d7f..0a056ee4 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -14,6 +14,11 @@ from collections import OrderedDict from contextlib import contextmanager from ftplib import FTP + +try: + from ftplib import FTP_TLS +except ImportError: + FTP_TLS = None from ftplib import error_perm from ftplib import error_temp from typing import cast @@ -346,7 +351,30 @@ def seek(self, pos, whence=Seek.set): class FTPFS(FS): - """A FTP (File Transport Protocol) Filesystem.""" + """A FTP (File Transport Protocol) Filesystem. + + Optionally, the connection can be made securely via TLS. This is known as + FTPS, or FTP Secure. TLS will be enabled when using the ftps:// protocol, + or when setting the `tls` argument to True in the constructor. + + + Examples: + Create with the constructor:: + + >>> from fs.ftpfs import FTPFS + >>> ftp_fs = FTPFS() + + Or via an FS URL:: + + >>> import fs + >>> ftp_fs = fs.open_fs('ftp://') + + Or via an FS URL, using TLS:: + + >>> import fs + >>> ftp_fs = fs.open_fs('ftps://') + + """ _meta = { "invalid_path_chars": "\0", @@ -366,6 +394,7 @@ def __init__( timeout=10, # type: int port=21, # type: int proxy=None, # type: Optional[Text] + tls=False, # type: bool ): # type: (...) -> None """Create a new `FTPFS` instance. @@ -380,6 +409,7 @@ def __init__( port (int): FTP port number (default 21). proxy (str, optional): An FTP proxy, or ``None`` (default) for no proxy. + tls (bool): Attempt to use FTP over TLS (FTPS) (default: False) """ super(FTPFS, self).__init__() @@ -390,6 +420,7 @@ def __init__( self.timeout = timeout self.port = port self.proxy = proxy + self.tls = tls self.encoding = "latin-1" self._ftp = None # type: Optional[FTP] @@ -432,11 +463,17 @@ def _parse_features(cls, feat_response): def _open_ftp(self): # type: () -> FTP """Open a new ftp object.""" - _ftp = FTP() + if self.tls and FTP_TLS: + _ftp = FTP_TLS() + else: + self.tls = False + _ftp = FTP() _ftp.set_debuglevel(0) with ftp_errors(self): _ftp.connect(self.host, self.port, self.timeout) _ftp.login(self.user, self.passwd, self.acct) + if self.tls: + _ftp.prot_p() self._features = {} try: feat_response = _decode(_ftp.sendcmd("FEAT"), "latin-1") @@ -471,7 +508,9 @@ def ftp_url(self): _user_part = "" else: _user_part = "{}:{}@".format(self.user, self.passwd) - url = "ftp://{}{}".format(_user_part, _host_part) + + scheme = "ftps" if self.tls else "ftp" + url = "{}://{}{}".format(scheme, _user_part, _host_part) return url @property From e19819b4599935eff269d959a1d07aa08e7e316a Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 17:17:13 +0000 Subject: [PATCH 100/309] FTPFS: Add FTPS support to the FTPFS opener. Enable TLS when using the ftps:// protocol --- fs/opener/ftpfs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/opener/ftpfs.py b/fs/opener/ftpfs.py index af64606b..cd313a34 100644 --- a/fs/opener/ftpfs.py +++ b/fs/opener/ftpfs.py @@ -23,7 +23,7 @@ class FTPOpener(Opener): """`FTPFS` opener.""" - protocols = ["ftp"] + protocols = ["ftp", "ftps"] @CreateFailed.catch_all def open_fs( @@ -48,6 +48,7 @@ def open_fs( passwd=parse_result.password, proxy=parse_result.params.get("proxy"), timeout=int(parse_result.params.get("timeout", "10")), + tls=bool(parse_result.protocol == "ftps"), ) if dir_path: if create: From 2abeac807d12c84239de5f3c3c8beabef04a90df Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 17:56:18 +0000 Subject: [PATCH 101/309] FTPFS: Add FTPS check to test case. --- tests/test_ftpfs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index fd34c134..b414283d 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -88,6 +88,10 @@ def test_opener(self): self.assertIsInstance(ftp_fs, FTPFS) self.assertEqual(ftp_fs.host, "ftp.example.org") + ftps_fs = open_fs("ftps://will:wfc@ftp.example.org") + self.assertIsInstance(ftps_fs, FTPFS) + self.assertTrue(ftps_fs.tls) + class TestFTPErrors(unittest.TestCase): """Test the ftp_errors context manager.""" From 5d145ccd29e901c3dbe1c331fe04b141a1aa39e8 Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 18:15:57 +0000 Subject: [PATCH 102/309] FTPFS: Fix opener test, and add extra test for the FTPS opener. --- tests/test_opener.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_opener.py b/tests/test_opener.py index fde555a2..e3bfd666 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -300,7 +300,11 @@ def test_user_data_opener(self, app_dir): def test_open_ftp(self, mock_FTPFS): open_fs("ftp://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( - "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10 + "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=False + ) + open_fs("ftps://foo:bar@ftp.example.org") + mock_FTPFS.assert_called_once_with( + "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=True ) @mock.patch("fs.ftpfs.FTPFS") @@ -313,4 +317,5 @@ def test_open_ftp_proxy(self, mock_FTPFS): user="foo", proxy="ftp.proxy.org", timeout=10, + tls=False, ) From 2d637f580c397c6d057a430fbff261fca63099b2 Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 18:41:06 +0000 Subject: [PATCH 103/309] FTPFS: Raise error in constructor if TLS is enabled but FTP_TLS is not available. --- fs/ftpfs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 0a056ee4..a05757f1 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -17,14 +17,15 @@ try: from ftplib import FTP_TLS -except ImportError: - FTP_TLS = None +except ImportError as err: + FTP_TLS = err from ftplib import error_perm from ftplib import error_temp from typing import cast from six import PY2 from six import text_type +from six import raise_from from . import errors from .base import FS @@ -422,6 +423,9 @@ def __init__( self.proxy = proxy self.tls = tls + if self.tls and isinstance(FTP_TLS, Exception): + raise_from(errors.CreateFailed("FTP over TLS not supported"), FTP_TLS) + self.encoding = "latin-1" self._ftp = None # type: Optional[FTP] self._welcome = None # type: Optional[Text] @@ -463,11 +467,7 @@ def _parse_features(cls, feat_response): def _open_ftp(self): # type: () -> FTP """Open a new ftp object.""" - if self.tls and FTP_TLS: - _ftp = FTP_TLS() - else: - self.tls = False - _ftp = FTP() + _ftp = FTP_TLS() if self.tls else FTP() _ftp.set_debuglevel(0) with ftp_errors(self): _ftp.connect(self.host, self.port, self.timeout) From 7609f1b668afb3b7fd8d51298e040fe094e8d578 Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 18:46:22 +0000 Subject: [PATCH 104/309] FTPFS: Test fix - move opener unit test for FTPS into its own function. --- tests/test_opener.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_opener.py b/tests/test_opener.py index e3bfd666..e7fae983 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -302,6 +302,9 @@ def test_open_ftp(self, mock_FTPFS): mock_FTPFS.assert_called_once_with( "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=False ) + + @mock.patch("fs.ftpfs.FTPFS") + def test_open_ftps(self, mock_FTPFS): open_fs("ftps://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=True From 831979ccc6597fc8a906a23b006155433eb0694f Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 18:57:41 +0000 Subject: [PATCH 105/309] Add to FTPS changes to Changelog, add myself to Contributors. --- CHANGELOG.md | 6 ++++++ CONTRIBUTORS.md | 1 + 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b75569fb..31cb320c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### Added + +- Added FTP over TLS (FTPS) support to FTPFS. + Closes [#437](https://github.com/PyFilesystem/pyfilesystem2/issues/437), + [#449](https://github.com/PyFilesystem/pyfilesystem2/pull/449). + ### Changed - Make `FS.upload` explicit about the expected error when the parent directory of the destination does not exist. diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 3c2729c7..46408d87 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -13,5 +13,6 @@ Many thanks to the following developers for contributing to this project: - [Martin Larralde](https://github.com/althonos) - [Morten Engelhardt Olsen](https://github.com/xoriath) - [Nick Henderson](https://github.com/nwh) +- [Oliver Galvin](https://github.com/odgalvin) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) From 89d9f7bbfc44022705fb3a49d1c63d435ec8de3f Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 19:07:35 +0000 Subject: [PATCH 106/309] FTPFS: Type checker fixes. --- fs/ftpfs.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index a05757f1..c28bc2e6 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -18,7 +18,7 @@ try: from ftplib import FTP_TLS except ImportError as err: - FTP_TLS = err + FTP_TLS = err # type: ignore from ftplib import error_perm from ftplib import error_temp from typing import cast @@ -472,8 +472,10 @@ def _open_ftp(self): with ftp_errors(self): _ftp.connect(self.host, self.port, self.timeout) _ftp.login(self.user, self.passwd, self.acct) - if self.tls: + try: _ftp.prot_p() + except AttributeError: + pass self._features = {} try: feat_response = _decode(_ftp.sendcmd("FEAT"), "latin-1") From e2c8bec72e7639f059fb6343ca3fb00678dc1830 Mon Sep 17 00:00:00 2001 From: Oliver Galvin Date: Sun, 31 Jan 2021 19:10:01 +0000 Subject: [PATCH 107/309] FTPFS: Type checker fixes. --- fs/ftpfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index c28bc2e6..3d8b0944 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -473,7 +473,7 @@ def _open_ftp(self): _ftp.connect(self.host, self.port, self.timeout) _ftp.login(self.user, self.passwd, self.acct) try: - _ftp.prot_p() + _ftp.prot_p() # type: ignore except AttributeError: pass self._features = {} From 43c807c26dc0ff44cec5749962c92acd7f251036 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 31 Jan 2021 22:34:14 +0100 Subject: [PATCH 108/309] Add changelog and contribution files to the sdist archive --- MANIFEST.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 99db56b1..cf5499b5 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,6 @@ +include CHANGELOG.md +include CONTRIBUTING.md +include CONTRIBUTORS.md include LICENSE graft tests graft docs From f0f45dc240da0dcf9b98bad4271d52a9d230f1ea Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 1 Feb 2021 03:51:01 +0100 Subject: [PATCH 109/309] Improve contents of `CONTRIBUTING.md` listing all `tox` environments --- CONTRIBUTING.md | 118 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e84f7586..5347176f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,14 +2,122 @@ Pull Requests are very welcome for this project! -For bug fixes or new features, please file an issue before submitting a pull request. If the change isn't trivial, it may be best to wait for feedback. For a quicker response, contact [Will McGugan](mailto:willmcgugan+pyfs@gmail.com) directly. +For bug fixes or new features, please file an issue before submitting a pull +request. If the change isn't trivial, it may be best to wait for feedback. +For a quicker response, contact [Will McGugan](mailto:willmcgugan+pyfs@gmail.com) +directly. -## Coding Guidelines -This project runs on Python2.7 and Python3.X. Python2.7 will be dropped at some point, but for now, please maintain compatibility. +## `tox` + +Most of the guidelines that follow can be checked with a particular +[`tox`](https://pypi.org/project/tox/) environment. Having it installed will +help you develop and verify your code locally without having to wait for +our Continuous Integration pipeline to finish. -Please format new code with [black](https://github.com/ambv/black), using the default settings. ## Tests -New code should have unit tests. We strive to have near 100% coverage. Get in touch, if you need assistance with the tests. +New code should have unit tests. We strive to have near 100% coverage. +Get in touch, if you need assistance with the tests. You shouldn't refrain +from opening a Pull Request even if all the tests were not added yet, or if +not all of them are passing yet. + +### Dependencies + +The dependency for running the tests can be found in the `tests/requirements.txt` file. +If you're using `tox`, you won't have to install them manually. Otherwise, +they can be installed with `pip`: +```console +$ pip install -r tests/requirements.txt +``` + +### Running (with `tox`) + +Simply run in the repository folder to execute the tests for all available +environments: +```console +$ tox +``` + +Since this can take some time, you can use a single environment to run +tests only once, for instance to run tests only with Python 3.9: +```console +$ tox -e py39 +``` + +### Running (without `tox`) + +Tests are written using the standard [`unittest`](https://docs.python.org/3/library/unittest.html) +framework. You should be able to run them using the standard library runner: +```console +$ python -m unittest discover -vv +``` + + +## Coding Guidelines + +This project runs on Python2.7 and Python3.X. Python2.7 will be dropped at +some point, but for now, please maintain compatibility. PyFilesystem2 uses +the [`six`](https://pypi.org/project/six/) library to write version-agnostic +Python code. + +### Style + +The code (including the tests) should follow PEP8. You can check for the +code style with: +```console +$ tox -e codestyle +``` + +This will invoke [`flake8`](https://pypi.org/project/flake8/) with some common +plugins such as [`flake8-comprehensions`](https://pypi.org/project/flake8-comprehensions/). + +### Format + +Please format new code with [black](https://github.com/ambv/black), using the +default settings. You can check whether the code is well-formatted with: +```console +$ tox -e codeformat +``` + +### Type annotations + +The code is typechecked with [`mypy`](https://pypi.org/project/mypy/), and +type annotations written as comments, to stay compatible with Python2. Run +the typechecking with: +```console +$ tox -e typecheck +``` + + +## Documentation + +### Dependencies + +The documentation is built with [Sphinx](https://pypi.org/project/Sphinx/), +using the [ReadTheDocs](https://pypi.org/project/sphinx-rtd-theme/) theme. +The dependencies are listed in `docs/requirements.txt` and can be installed with +`pip`: +```console +$ pip install -r docs/requirements.txt +``` + +### Building + +Run the following command to build the HTML documentation: +```console +$ python setup.py build_sphinx +``` + +The documentation index will be written to the `build/sphinx/html/` +directory. + +### Style + +The API reference is written in the Python source, using docstrings in +[Google format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). +The documentation style can be checked with: +```console +$ tox -e docstyle +``` From 8a6686f374637803cba12212ee8fa5203730d7d4 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 1 Feb 2021 15:23:44 +0100 Subject: [PATCH 110/309] Add all GitHub contributors to `CONTRIBUTORS.md` --- CONTRIBUTORS.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 46408d87..2f8ec34f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -2,17 +2,42 @@ Many thanks to the following developers for contributing to this project: +- [Alex Povel](https://github.com/alexpovel) - [Andreas Tollkötter](https://github.com/atollk) -- [C. W.](https://github.com/chfw) +- [Andrew Scheller](https://github.com/lurch) +- [Andrey Serov](https://github.com/zmej-serow) +- [Bernhard M. Wiedemann](https://github.com/bmwiedemann) +- [@chfw](https://github.com/chfw) +- [Dafna Hirschfeld](https://github.com/kamomil) - [Diego Argueta](https://github.com/dargueta) - [Eelke van den Bos](https://github.com/eelkevdbos) +- [Egor Namakonov](https://github.com/fresheed) +- [@FooBarQuaxx](https://github.com/FooBarQuaxx) - [Geoff Jukes](https://github.com/geoffjukes) +- [George Macon](https://github.com/gmacon) - [Giampaolo Cimino](https://github.com/gpcimino) +- [@Hoboneer](https://github.com/Hoboneer) +- [Joseph Atkins-Turkish](https://github.com/Spacerat) +- [Joshua Tauberer](https://github.com/JoshData) - [Justin Charlong](https://github.com/jcharlong) - [Louis Sautier](https://github.com/sbraz) +- [Martin Durant](https://github.com/martindurant) - [Martin Larralde](https://github.com/althonos) +- [Masaya Nakamura](https://github.com/mashabow) - [Morten Engelhardt Olsen](https://github.com/xoriath) +- [@mrg0029](https://github.com/mrg0029) +- [Nathan Goldbaum](https://github.com/ngoldbaum) - [Nick Henderson](https://github.com/nwh) - [Oliver Galvin](https://github.com/odgalvin) +- [Philipp Wiesner](https://github.com/birnbaum) +- [Philippe Ombredanne](https://github.com/pombredanne) +- [Rehan Khwaja](https://github.com/rkhwaja) +- [Silvan Spross](https://github.com/sspross) +- [@sqwishy](https://github.com/sqwishy) +- [Sven Schliesing](https://github.com/muffl0n) +- [Tim Gates](https://github.com/timgates42/) +- [@tkossak](https://github.com/tkossak) +- [Todd Levi](https://github.com/televi) +- [Vilius Grigaliūnas](https://github.com/vilius-g) - [Will McGugan](https://github.com/willmcgugan) - [Zmej Serow](https://github.com/zmej-serow) From 65eff7cfc41e536cf3b529cc34d3b94ab38a4cc8 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 5 Feb 2021 16:17:25 +0100 Subject: [PATCH 111/309] Add classifier marking the project as typed to `setup.cfg` --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index faad103c..b90ebe47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ classifiers = Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: System :: Filesystems + Typing :: Typed project_urls = Bug Reports = https://github.com/PyFilesystem/pyfilesystem2/issues Documentation = https://pyfilesystem2.readthedocs.io/en/latest/ From 64d7a52e1f3037f33937f91e9db812cafdbbafc0 Mon Sep 17 00:00:00 2001 From: Andrew Scheller Date: Fri, 12 Mar 2021 12:46:07 +0000 Subject: [PATCH 112/309] Docs typo --- fs/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index ecd82a9e..7a81c5cb 100644 --- a/fs/base.py +++ b/fs/base.py @@ -796,7 +796,7 @@ def getospath(self, path): Note: If you want your code to work in Python2.7 and Python3 then - use this method if you want to work will the OS filesystem + use this method if you want to work with the OS filesystem outside of the OSFS interface. """ From 72e2fc9f5d552b32f91ea3ee92f6c8cf9c4899c8 Mon Sep 17 00:00:00 2001 From: atollk Date: Fri, 19 Mar 2021 09:12:46 +0100 Subject: [PATCH 113/309] In fs.ftpfs.FTPFS.upload, replaced `self._manage_ftp` by `self.ftp`. See issue 455 for a detailed discussion. --- fs/ftpfs.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 3d8b0944..c5929ee1 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -814,11 +814,10 @@ def upload(self, path, file, chunk_size=None, **options): # type: (Text, BinaryIO, Optional[int], **Any) -> None _path = self.validatepath(path) with self._lock: - with self._manage_ftp() as ftp: - with ftp_errors(self, path): - ftp.storbinary( - str("STOR ") + _encode(_path, self.ftp.encoding), file - ) + with ftp_errors(self, path): + self.ftp.storbinary( + str("STOR ") + _encode(_path, self.ftp.encoding), file + ) def writebytes(self, path, contents): # type: (Text, ByteString) -> None From 341bf7a84bc59677ad3f0a25a05e2d048302aca0 Mon Sep 17 00:00:00 2001 From: atollk Date: Fri, 19 Mar 2021 22:21:49 +0100 Subject: [PATCH 114/309] Added _copy_if functions to fs.copy module. The free functions copy_file, copy_file_if_newer, copy_dir, copy_dir_if_newer, copy_fs, and copy_fs_if_newer now are all implemented in terms of copy_file_if, copy_dir_if, and copy_fs_if. Unit Tests for this change will be created in a future commit. This change streamlined the code by removing duplication in logic and clustering similar logic closer together. Also, though this has not been tested, copy_dir_if_newer should be faster in this implementation when copying to a dst_dir that contains a lot of files that are not present in the respective src_dir. Finally, a bug was fixed that was caused by a false lookup in the dictionary of file infos from dst_fs. This fix could cause unnecessary copies and therefore a decrease in performance. --- fs/copy.py | 417 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 261 insertions(+), 156 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 6cd34392..3f949cc3 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -5,7 +5,7 @@ import typing -from .errors import FSError +from .errors import ResourceNotFound from .opener import manage_fs from .path import abspath, combine, frombase, normpath from .tools import is_thread_safe @@ -41,9 +41,7 @@ def copy_fs( a single-threaded copy. """ - return copy_dir( - src_fs, "/", dst_fs, "/", walker=walker, on_copy=on_copy, workers=workers - ) + return copy_fs_if(src_fs, dst_fs, "always", walker, on_copy, workers) def copy_fs_if_newer( @@ -74,38 +72,61 @@ def copy_fs_if_newer( a single-threaded copy. """ - return copy_dir_if_newer( - src_fs, "/", dst_fs, "/", walker=walker, on_copy=on_copy, workers=workers - ) + return copy_fs_if(src_fs, dst_fs, "newer", walker, on_copy, workers) -def _source_is_newer(src_fs, src_path, dst_fs, dst_path): - # type: (FS, Text, FS, Text) -> bool - """Determine if source file is newer than destination file. +def copy_fs_if( + src_fs, # type: Union[FS, Text] + dst_fs, # type: Union[FS, Text] + condition="always", # type: Text + walker=None, # type: Optional[Walker] + on_copy=None, # type: Optional[_OnCopy] + workers=0, # type: int +): + # type: (...) -> None + """Copy the contents of one filesystem to another, depending on a condition. - Arguments: - src_fs (FS): Source filesystem (instance or URL). - src_path (str): Path to a file on the source filesystem. - dst_fs (FS): Destination filesystem (instance or URL). - dst_path (str): Path to a file on the destination filesystem. + Depending on the value of ``strategy``, certain conditions must be fulfilled + for a file to be copied to ``dst_fs``. - Returns: - bool: `True` if the source file is newer than the destination - file or file modification time cannot be determined, `False` - otherwise. + If ``condition`` has the value ``"newer"``, the last modification time + of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"older"``, the last modification time + of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"exists"``, the source file is only + copied if a file of the same path already exists in ``dst_fs``. + + If ``condition`` has the value ``"not_exists"``, the source file is only + copied if no file of the same path already exists in ``dst_fs``. + + Arguments: + src_fs (FS or str): Source filesystem (URL or instance). + dst_fs (FS or str): Destination filesystem (URL or instance). + condition (str): Name of the condition to check for each file. + walker (~fs.walk.Walker, optional): A walker object that will be + used to scan for files in ``src_fs``. Set this if you only want + to consider a sub-set of the resources in ``src_fs``. + on_copy (callable):A function callback called after a single file copy + is executed. Expected signature is ``(src_fs, src_path, dst_fs, + dst_path)``. + workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for + a single-threaded copy. """ - try: - if dst_fs.exists(dst_path): - namespace = ("details", "modified") - src_modified = src_fs.getinfo(src_path, namespace).modified - if src_modified is not None: - dst_modified = dst_fs.getinfo(dst_path, namespace).modified - return dst_modified is None or src_modified > dst_modified - return True - except FSError: # pragma: no cover - # todo: should log something here - return True + return copy_dir_if( + src_fs, + "/", + dst_fs, + "/", + condition, + walker=walker, + on_copy=on_copy, + workers=workers, + ) def copy_file( @@ -126,76 +147,70 @@ def copy_file( dst_path (str): Path to a file on the destination filesystem. """ - with manage_fs(src_fs, writeable=False) as _src_fs: - with manage_fs(dst_fs, create=True) as _dst_fs: - if _src_fs is _dst_fs: - # Same filesystem, so we can do a potentially optimized - # copy - _src_fs.copy(src_path, dst_path, overwrite=True) - else: - # Standard copy - with _src_fs.lock(), _dst_fs.lock(): - if _dst_fs.hassyspath(dst_path): - with _dst_fs.openbin(dst_path, "w") as write_file: - _src_fs.download(src_path, write_file) - else: - with _src_fs.openbin(src_path) as read_file: - _dst_fs.upload(dst_path, read_file) + copy_file_if(src_fs, src_path, dst_fs, dst_path, "always") -def copy_file_internal( - src_fs, # type: FS +def copy_file_if_newer( + src_fs, # type: Union[FS, Text] src_path, # type: Text - dst_fs, # type: FS + dst_fs, # type: Union[FS, Text] dst_path, # type: Text ): - # type: (...) -> None - """Copy a file at low level, without calling `manage_fs` or locking. + # type: (...) -> bool + """Copy a file from one filesystem to another, checking times. If the destination exists, and is a file, it will be first truncated. - - This method exists to optimize copying in loops. In general you - should prefer `copy_file`. + If both source and destination files exist, the copy is executed only + if the source file is newer than the destination file. In case + modification times of source or destination files are not available, + copy is always executed. Arguments: - src_fs (FS): Source filesystem. + src_fs (FS or str): Source filesystem (instance or URL). src_path (str): Path to a file on the source filesystem. - dst_fs (FS): Destination filesystem. + dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. + Returns: + bool: `True` if the file copy was executed, `False` otherwise. + """ - if src_fs is dst_fs: - # Same filesystem, so we can do a potentially optimized - # copy - src_fs.copy(src_path, dst_path, overwrite=True) - elif dst_fs.hassyspath(dst_path): - with dst_fs.openbin(dst_path, "w") as write_file: - src_fs.download(src_path, write_file) - else: - with src_fs.openbin(src_path) as read_file: - dst_fs.upload(dst_path, read_file) + return copy_file_if(src_fs, src_path, dst_fs, dst_path, "newer") -def copy_file_if_newer( +def copy_file_if( src_fs, # type: Union[FS, Text] src_path, # type: Text dst_fs, # type: Union[FS, Text] dst_path, # type: Text + condition, # type: Text ): # type: (...) -> bool - """Copy a file from one filesystem to another, checking times. + """Copy a file from one filesystem to another, depending on a condition. - If the destination exists, and is a file, it will be first truncated. - If both source and destination files exist, the copy is executed only - if the source file is newer than the destination file. In case - modification times of source or destination files are not available, - copy is always executed. + Depending on the value of ``strategy``, certain conditions must be fulfilled + for a file to be copied to ``dst_fs``. + + If ``condition`` has the value ``"newer"``, the last modification time + of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"older"``, the last modification time + of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"exists"``, the source file is only + copied if a file of the same path already exists in ``dst_fs``. + + If ``condition`` has the value ``"not_exists"``, the source file is only + copied if no file of the same path already exists in ``dst_fs``. Arguments: src_fs (FS or str): Source filesystem (instance or URL). src_path (str): Path to a file on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. + condition (str): Name of the condition to check for each file. Returns: bool: `True` if the file copy was executed, `False` otherwise. @@ -203,28 +218,64 @@ def copy_file_if_newer( """ with manage_fs(src_fs, writeable=False) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: - if _src_fs is _dst_fs: - # Same filesystem, so we can do a potentially optimized - # copy - if _source_is_newer(_src_fs, src_path, _dst_fs, dst_path): - _src_fs.copy(src_path, dst_path, overwrite=True) - return True - else: - return False - else: - # Standard copy - with _src_fs.lock(), _dst_fs.lock(): - if _source_is_newer(_src_fs, src_path, _dst_fs, dst_path): - copy_file_internal(_src_fs, src_path, _dst_fs, dst_path) - return True - else: - return False + do_copy = _copy_is_necessary( + _src_fs, src_path, _dst_fs, dst_path, condition + ) + if do_copy: + copy_file_internal(_src_fs, src_path, _dst_fs, dst_path, True) + return do_copy + + +def copy_file_internal( + src_fs, # type: FS + src_path, # type: Text + dst_fs, # type: FS + dst_path, # type: Text + lock=False, # type: bool +): + # type: (...) -> None + """Copy a file at low level, without calling `manage_fs` or locking. + + If the destination exists, and is a file, it will be first truncated. + + This method exists to optimize copying in loops. In general you + should prefer `copy_file`. + + Arguments: + src_fs (FS): Source filesystem. + src_path (str): Path to a file on the source filesystem. + dst_fs (FS): Destination filesystem. + dst_path (str): Path to a file on the destination filesystem. + lock (bool): Lock both filesystems before copying. + + """ + if src_fs is dst_fs: + # Same filesystem, so we can do a potentially optimized + # copy + src_fs.copy(src_path, dst_path, overwrite=True) + return + + def _copy_locked(): + if dst_fs.hassyspath(dst_path): + with dst_fs.openbin(dst_path, "w") as write_file: + src_fs.download(src_path, write_file) + else: + with src_fs.openbin(src_path) as read_file: + dst_fs.upload(dst_path, read_file) + + if lock: + with src_fs.lock(), dst_fs.lock(): + _copy_locked() + else: + _copy_locked() def copy_structure( src_fs, # type: Union[FS, Text] dst_fs, # type: Union[FS, Text] walker=None, # type: Optional[Walker] + src_root="/", # type: Text + dst_root="/", # type: Text ): # type: (...) -> None """Copy directories (but not files) from ``src_fs`` to ``dst_fs``. @@ -235,14 +286,20 @@ def copy_structure( walker (~fs.walk.Walker, optional): A walker object that will be used to scan for files in ``src_fs``. Set this if you only want to consider a sub-set of the resources in ``src_fs``. + src_root (str): Path of the base directory to consider as the root + of the tree structure to copy. + dst_root (str): Path to the target root of the tree structure. """ walker = walker or Walker() with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): - for dir_path in walker.dirs(_src_fs): - _dst_fs.makedir(dir_path, recreate=True) + _dst_fs.makedirs(dst_root, recreate=True) + for dir_path in walker.dirs(_src_fs, src_root): + _dst_fs.makedir( + combine(dst_root, frombase(src_root, dir_path)), recreate=True + ) def copy_dir( @@ -272,33 +329,7 @@ def copy_dir( a single-threaded copy. """ - on_copy = on_copy or (lambda *args: None) - walker = walker or Walker() - _src_path = abspath(normpath(src_path)) - _dst_path = abspath(normpath(dst_path)) - - def src(): - return manage_fs(src_fs, writeable=False) - - def dst(): - return manage_fs(dst_fs, create=True) - - from ._bulk import Copier - - with src() as _src_fs, dst() as _dst_fs: - with _src_fs.lock(), _dst_fs.lock(): - _thread_safe = is_thread_safe(_src_fs, _dst_fs) - with Copier(num_workers=workers if _thread_safe else 0) as copier: - _dst_fs.makedir(_dst_path, recreate=True) - for dir_path, dirs, files in walker.walk(_src_fs, _src_path): - copy_path = combine(_dst_path, frombase(_src_path, dir_path)) - for info in dirs: - _dst_fs.makedir(info.make_path(copy_path), recreate=True) - for info in files: - src_path = info.make_path(dir_path) - dst_path = info.make_path(copy_path) - copier.copy(_src_fs, src_path, _dst_fs, dst_path) - on_copy(_src_fs, src_path, _dst_fs, dst_path) + copy_dir_if(src_fs, src_path, dst_fs, dst_path, "always", walker, on_copy, workers) def copy_dir_if_newer( @@ -332,54 +363,128 @@ def copy_dir_if_newer( workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + """ + copy_dir_if(src_fs, src_path, dst_fs, dst_path, "newer", walker, on_copy, workers) + + +def copy_dir_if( + src_fs, # type: Union[FS, Text] + src_path, # type: Text + dst_fs, # type: Union[FS, Text] + dst_path, # type: Text + condition="always", # type: Text + walker=None, # type: Optional[Walker] + on_copy=None, # type: Optional[_OnCopy] + workers=0, # type: int +): + # type: (...) -> None + """Copy a directory from one filesystem to another, depending on a condition. + + Depending on the value of ``strategy``, certain conditions must be + fulfilled for a file to be copied to ``dst_fs``. + + If ``condition`` has the value ``"always"``, the source file is always + copied. + + If ``condition`` has the value ``"newer"``, the last modification time + of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"older"``, the last modification time + of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + + If ``condition`` has the value ``"exists"``, the source file is only + copied if a file of the same path already exists in ``dst_fs``. + + If ``condition`` has the value ``"not_exists"``, the source file is only + copied if no file of the same path already exists in ``dst_fs``. + + Arguments: + src_fs (FS or str): Source filesystem (instance or URL). + src_path (str): Path to a directory on the source filesystem. + dst_fs (FS or str): Destination filesystem (instance or URL). + dst_path (str): Path to a directory on the destination filesystem. + condition (str): Name of the condition to check for each file. + walker (~fs.walk.Walker, optional): A walker object that will be + used to scan for files in ``src_fs``. Set this if you only want + to consider a sub-set of the resources in ``src_fs``. + on_copy (callable):A function callback called after a single file copy + is executed. Expected signature is ``(src_fs, src_path, dst_fs, + dst_path)``. + workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for + a single-threaded copy. + """ on_copy = on_copy or (lambda *args: None) walker = walker or Walker() _src_path = abspath(normpath(src_path)) _dst_path = abspath(normpath(dst_path)) - def src(): - return manage_fs(src_fs, writeable=False) - - def dst(): - return manage_fs(dst_fs, create=True) - from ._bulk import Copier - with src() as _src_fs, dst() as _dst_fs: + copy_structure(src_fs, dst_fs, walker, src_path, dst_path) + + with manage_fs(src_fs, writeable=False) as _src_fs, manage_fs( + dst_fs, create=True + ) as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): _thread_safe = is_thread_safe(_src_fs, _dst_fs) with Copier(num_workers=workers if _thread_safe else 0) as copier: - _dst_fs.makedir(_dst_path, recreate=True) - namespace = ("details", "modified") - dst_state = { - path: info - for path, info in walker.info(_dst_fs, _dst_path, namespace) - if info.is_file - } - src_state = [ - (path, info) - for path, info in walker.info(_src_fs, _src_path, namespace) - ] - for dir_path, copy_info in src_state: + for dir_path in walker.files(_src_fs, _src_path): copy_path = combine(_dst_path, frombase(_src_path, dir_path)) - if copy_info.is_dir: - _dst_fs.makedir(copy_path, recreate=True) - elif copy_info.is_file: - # dst file is present, try to figure out if copy - # is necessary - try: - src_modified = copy_info.modified - dst_modified = dst_state[dir_path].modified - except KeyError: - do_copy = True - else: - do_copy = ( - src_modified is None - or dst_modified is None - or src_modified > dst_modified - ) - - if do_copy: - copier.copy(_src_fs, dir_path, _dst_fs, copy_path) - on_copy(_src_fs, dir_path, _dst_fs, copy_path) + if _copy_is_necessary( + _src_fs, dir_path, _dst_fs, copy_path, condition + ): + copier.copy(_src_fs, dir_path, _dst_fs, copy_path) + on_copy(_src_fs, dir_path, _dst_fs, copy_path) + + +def _copy_is_necessary( + src_fs, # type: FS + src_path, # type: Text + dst_fs, # type: FS + dst_path, # type: Text + condition, # type: Text +): + # type: (...) -> bool + + if condition == "always": + return True + + elif condition == "newer": + try: + namespace = ("details", "modified") + src_modified = src_fs.getinfo(src_path, namespace).modified + dst_modified = dst_fs.getinfo(dst_path, namespace).modified + except ResourceNotFound: + return True + else: + return ( + src_modified is None + or dst_modified is None + or src_modified > dst_modified + ) + + elif condition == "older": + try: + namespace = ("details", "modified") + src_modified = src_fs.getinfo(src_path, namespace).modified + dst_modified = dst_fs.getinfo(dst_path, namespace).modified + except ResourceNotFound: + return True + else: + return ( + src_modified is None + or dst_modified is None + or src_modified < dst_modified + ) + + elif condition == "exists": + return dst_fs.exists(dst_path) + + elif condition == "not_exists": + return not dst_fs.exists(dst_path) + + else: + raise ValueError(condition + "is not a valid copy condition.") From 6f3993d0a3d652cf96a1b528cc2f4dc9d43da77d Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 11:31:32 +0100 Subject: [PATCH 115/309] Implemented unit tests for the recently added _copy_if functions in fs.copy. tests/test_copy experienced a general rework in many areas. Before, tests focused on the most basic of test cases and thus missed more complex situations that could (and actually did) cause errors. Thus, some tests for the same unit were merged to create more relevant test cases. --- tests/test_copy.py | 878 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 745 insertions(+), 133 deletions(-) diff --git a/tests/test_copy.py b/tests/test_copy.py index 63e550e9..92d77c33 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -12,7 +12,55 @@ from fs import open_fs -class TestCopy(unittest.TestCase): +def _create_sandbox_dir(prefix="pyfilesystem2_sandbox_", home=None): + if home is None: + return tempfile.mkdtemp(prefix=prefix) + else: + sandbox_path = os.path.join(home, prefix) + mkdirp(sandbox_path) + return sandbox_path + + +def _touch(root, filepath): + # create abs filename + abs_filepath = os.path.join(root, filepath) + # create dir + dirname = os.path.dirname(abs_filepath) + mkdirp(dirname) + # touch file + with open(abs_filepath, "a"): + os.utime( + abs_filepath, None + ) # update the mtime in case the file exists, same as touch + + return abs_filepath + + +def _write_file(filepath, write_chars=1024): + with open(filepath, "w") as f: + f.write("1" * write_chars) + return filepath + + +def _delay_file_utime(filepath, delta_sec): + utcnow = datetime.datetime.utcnow() + unix_timestamp = calendar.timegm(utcnow.timetuple()) + times = unix_timestamp + delta_sec, unix_timestamp + delta_sec + os.utime(filepath, times) + + +def mkdirp(path): + # os.makedirs(path, exist_ok=True) only for python3.? + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + +class TestCopySimple(unittest.TestCase): def test_copy_fs(self): for workers in (0, 1, 2, 4): src_fs = open_fs("mem://") @@ -78,50 +126,11 @@ def on_copy(*args): fs.copy.copy_dir(src_fs, "/", dst_fs, "/", on_copy=on_copy) self.assertEqual(on_copy_calls, [(src_fs, "/baz.txt", dst_fs, "/baz.txt")]) - def mkdirp(self, path): - # os.makedirs(path, exist_ok=True) only for python3.? - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - def _create_sandbox_dir(self, prefix="pyfilesystem2_sandbox_", home=None): - if home is None: - return tempfile.mkdtemp(prefix=prefix) - else: - sandbox_path = os.path.join(home, prefix) - self.mkdirp(sandbox_path) - return sandbox_path - - def _touch(self, root, filepath): - # create abs filename - abs_filepath = os.path.join(root, filepath) - # create dir - dirname = os.path.dirname(abs_filepath) - self.mkdirp(dirname) - # touch file - with open(abs_filepath, "a"): - os.utime( - abs_filepath, None - ) # update the mtime in case the file exists, same as touch - - return abs_filepath - - def _write_file(self, filepath, write_chars=1024): - with open(filepath, "w") as f: - f.write("1" * write_chars) - return filepath - - def _delay_file_utime(self, filepath, delta_sec): - utcnow = datetime.datetime.utcnow() - unix_timestamp = calendar.timegm(utcnow.timetuple()) - times = unix_timestamp + delta_sec, unix_timestamp + delta_sec - os.utime(filepath, times) - - def test_copy_file_if_newer_same_fs(self): + +class TestCopyIfNewer(unittest.TestCase): + copy_if_condition = "newer" + + def test_copy_file_if_same_fs(self): src_fs = open_fs("mem://") src_fs.makedir("foo2").touch("exists") src_fs.makedir("foo1").touch("test1.txt") @@ -129,35 +138,42 @@ def test_copy_file_if_newer_same_fs(self): "foo2/exists", datetime.datetime.utcnow() + datetime.timedelta(hours=1) ) self.assertTrue( - fs.copy.copy_file_if_newer( - src_fs, "foo1/test1.txt", src_fs, "foo2/test1.txt.copy" + fs.copy.copy_file_if( + src_fs, + "foo1/test1.txt", + src_fs, + "foo2/test1.txt.copy", + self.copy_if_condition, ) ) self.assertFalse( - fs.copy.copy_file_if_newer(src_fs, "foo1/test1.txt", src_fs, "foo2/exists") + fs.copy.copy_file_if( + src_fs, "foo1/test1.txt", src_fs, "foo2/exists", self.copy_if_condition + ) ) self.assertTrue(src_fs.exists("foo2/test1.txt.copy")) - def test_copy_file_if_newer_dst_older(self): + def test_copy_file_if_dst_is_older(self): try: # create first dst ==> dst is older the src ==> file should be copied - dst_dir = self._create_sandbox_dir() - dst_file1 = self._touch(dst_dir, "file1.txt") - self._write_file(dst_file1) + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) # ensure src file is newer than dst, changing its modification time - self._delay_file_utime(src_file1, delta_sec=60) + _delay_file_utime(src_file1, delta_sec=60) src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) self.assertTrue(dst_fs.exists("/file1.txt")) - copied = fs.copy.copy_file_if_newer( - src_fs, "/file1.txt", dst_fs, "/file1.txt" + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition ) self.assertTrue(copied) @@ -166,19 +182,19 @@ def test_copy_file_if_newer_dst_older(self): shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_file_if_newer_dst_doesnt_exists(self): + def test_copy_file_if_dst_doesnt_exists(self): try: - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) - dst_dir = self._create_sandbox_dir() + dst_dir = _create_sandbox_dir() src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) - copied = fs.copy.copy_file_if_newer( - src_fs, "/file1.txt", dst_fs, "/file1.txt" + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition ) self.assertTrue(copied) @@ -187,57 +203,320 @@ def test_copy_file_if_newer_dst_doesnt_exists(self): shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_file_if_newer_dst_is_newer(self): + def test_copy_file_if_dst_is_newer(self): try: - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + # ensure dst file is newer than src, changing its modification time + _delay_file_utime(dst_file1, delta_sec=60) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertFalse(copied) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) - dst_dir = self._create_sandbox_dir() - dst_file1 = self._touch(dst_dir, "file1.txt") - self._write_file(dst_file1) + def test_copy_fs_if(self): + try: + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + dst_file2 = _touch(dst_dir, "file2.txt") + _write_file(dst_file1) + _write_file(dst_file2) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + src_file2 = _touch(src_dir, "file2.txt") + src_file3 = _touch(src_dir, "file3.txt") + _write_file(src_file1) + _write_file(src_file2) + _write_file(src_file3) + + # ensure src_file1 is newer than dst_file1, changing its modification time + # ensure dst_file2 is newer than src_file2, changing its modification time + _delay_file_utime(src_file1, delta_sec=60) + _delay_file_utime(dst_file2, delta_sec=60) src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertTrue(dst_fs.exists("/file2.txt")) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_fs_if( + src_fs, dst_fs, on_copy=on_copy, condition=self.copy_if_condition + ) + + self.assertTrue("/file1.txt" in copied) + self.assertTrue("/file2.txt" not in copied) + self.assertTrue("/file3.txt" in copied) + self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertTrue(dst_fs.exists("/file2.txt")) + self.assertTrue(dst_fs.exists("/file3.txt")) + + src_fs.close() + dst_fs.close() + + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_dir_if(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + src_file2 = _touch(src_dir, os.path.join("one_level_down", "file2.txt")) + _write_file(src_file2) + + dst_dir = _create_sandbox_dir() + mkdirp(os.path.join(dst_dir, "target_dir")) + dst_file1 = _touch(dst_dir, os.path.join("target_dir", "file1.txt")) + _write_file(dst_file1) + + # ensure dst file is newer than src, changing its modification time + _delay_file_utime(dst_file1, delta_sec=60) - copied = fs.copy.copy_file_if_newer( - src_fs, "/file1.txt", dst_fs, "/file1.txt" + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, + "/", + dst_fs, + "/target_dir/", + on_copy=on_copy, + condition=self.copy_if_condition, ) - self.assertEqual(copied, False) + self.assertEqual(copied, ["/target_dir/one_level_down/file2.txt"]) + self.assertTrue(dst_fs.exists("/target_dir/one_level_down/file2.txt")) + + src_fs.close() + dst_fs.close() finally: shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_fs_if_newer_dst_older(self): + def test_copy_dir_if_same_fs(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "src" + os.sep + "file1.txt") + _write_file(src_file1) + + _create_sandbox_dir(home=src_dir) + + src_fs = open_fs("osfs://" + src_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, "/src", src_fs, "/dst", on_copy=on_copy, condition="newer" + ) + + self.assertEqual(copied, ["/dst/file1.txt"]) + self.assertTrue(src_fs.exists("/dst/file1.txt")) + + src_fs.close() + + finally: + shutil.rmtree(src_dir) + + def test_copy_dir_if_multiple_files(self): + try: + src_dir = _create_sandbox_dir() + src_fs = open_fs("osfs://" + src_dir) + src_fs.makedirs("foo/bar") + src_fs.makedirs("foo/empty") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + + dst_dir = _create_sandbox_dir() + dst_fs = open_fs("osfs://" + dst_dir) + + fs.copy.copy_dir_if(src_fs, "/foo", dst_fs, "/", condition="newer") + + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isdir("empty")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + +class TestCopyIfOlder(unittest.TestCase): + copy_if_condition = "older" + + def test_copy_file_if_same_fs(self): + src_fs = open_fs("mem://") + src_fs.makedir("foo2").touch("exists") + src_fs.makedir("foo1").touch("test1.txt") + src_fs.settimes( + "foo2/exists", datetime.datetime.utcnow() - datetime.timedelta(hours=1) + ) + self.assertTrue( + fs.copy.copy_file_if( + src_fs, + "foo1/test1.txt", + src_fs, + "foo2/test1.txt.copy", + self.copy_if_condition, + ) + ) + self.assertFalse( + fs.copy.copy_file_if( + src_fs, "foo1/test1.txt", src_fs, "foo2/exists", self.copy_if_condition + ) + ) + self.assertTrue(src_fs.exists("foo2/test1.txt.copy")) + + def test_copy_file_if_dst_is_older(self): try: # create first dst ==> dst is older the src ==> file should be copied - dst_dir = self._create_sandbox_dir() - dst_file1 = self._touch(dst_dir, "file1.txt") - self._write_file(dst_file1) + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) # ensure src file is newer than dst, changing its modification time - self._delay_file_utime(src_file1, delta_sec=60) + _delay_file_utime(src_file1, delta_sec=60) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertFalse(copied) + self.assertTrue(dst_fs.exists("/file1.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_file_if_dst_doesnt_exists(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertTrue(copied) self.assertTrue(dst_fs.exists("/file1.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_file_if_dst_is_newer(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + # ensure dst file is newer than src, changing its modification time + _delay_file_utime(dst_file1, delta_sec=60) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertTrue(copied) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_fs_if(self): + try: + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + dst_file2 = _touch(dst_dir, "file2.txt") + _write_file(dst_file1) + _write_file(dst_file2) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + src_file2 = _touch(src_dir, "file2.txt") + src_file3 = _touch(src_dir, "file3.txt") + _write_file(src_file1) + _write_file(src_file2) + _write_file(src_file3) + + # ensure src_file1 is newer than dst_file1, changing its modification time + # ensure dst_file2 is newer than src_file2, changing its modification time + _delay_file_utime(src_file1, delta_sec=60) + _delay_file_utime(dst_file2, delta_sec=60) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertTrue(dst_fs.exists("/file2.txt")) copied = [] def on_copy(src_fs, src_path, dst_fs, dst_path): copied.append(dst_path) - fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy) + fs.copy.copy_fs_if( + src_fs, dst_fs, on_copy=on_copy, condition=self.copy_if_condition + ) - self.assertEqual(copied, ["/file1.txt"]) + self.assertTrue("/file1.txt" not in copied) + self.assertTrue("/file2.txt" in copied) + self.assertTrue("/file3.txt" in copied) self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertTrue(dst_fs.exists("/file2.txt")) + self.assertTrue(dst_fs.exists("/file3.txt")) src_fs.close() dst_fs.close() @@ -246,16 +525,22 @@ def on_copy(src_fs, src_path, dst_fs, dst_path): shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_fs_if_newer_when_dst_doesnt_exists(self): + def test_copy_dir_if(self): try: - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + src_file2 = _touch(src_dir, os.path.join("one_level_down", "file2.txt")) + _write_file(src_file2) - src_file2 = self._touch(src_dir, "one_level_down" + os.sep + "file2.txt") - self._write_file(src_file2) + dst_dir = _create_sandbox_dir() + mkdirp(os.path.join(dst_dir, "target_dir")) + dst_file1 = _touch(dst_dir, os.path.join("target_dir", "file1.txt")) + _write_file(dst_file1) - dst_dir = self._create_sandbox_dir() + # ensure src file is newer than dst, changing its modification time + _delay_file_utime(src_file1, delta_sec=60) src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) @@ -265,31 +550,152 @@ def test_copy_fs_if_newer_when_dst_doesnt_exists(self): def on_copy(src_fs, src_path, dst_fs, dst_path): copied.append(dst_path) - fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy) + fs.copy.copy_dir_if( + src_fs, + "/", + dst_fs, + "/target_dir/", + on_copy=on_copy, + condition=self.copy_if_condition, + ) - self.assertEqual(copied, ["/file1.txt", "/one_level_down/file2.txt"]) - self.assertTrue(dst_fs.exists("/file1.txt")) - self.assertTrue(dst_fs.exists("/one_level_down/file2.txt")) + self.assertEqual(copied, ["/target_dir/one_level_down/file2.txt"]) + self.assertTrue(dst_fs.exists("/target_dir/one_level_down/file2.txt")) src_fs.close() dst_fs.close() + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_dir_if_same_fs(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "src" + os.sep + "file1.txt") + _write_file(src_file1) + + _create_sandbox_dir(home=src_dir) + + src_fs = open_fs("osfs://" + src_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, "/src", src_fs, "/dst", on_copy=on_copy, condition="newer" + ) + + self.assertEqual(copied, ["/dst/file1.txt"]) + self.assertTrue(src_fs.exists("/dst/file1.txt")) + + src_fs.close() + + finally: + shutil.rmtree(src_dir) + + def test_copy_dir_if_multiple_files(self): + try: + src_dir = _create_sandbox_dir() + src_fs = open_fs("osfs://" + src_dir) + src_fs.makedirs("foo/bar") + src_fs.makedirs("foo/empty") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + + dst_dir = _create_sandbox_dir() + dst_fs = open_fs("osfs://" + dst_dir) + + fs.copy.copy_dir_if(src_fs, "/foo", dst_fs, "/", condition="newer") + + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isdir("empty")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + +class TestCopyIfExists(unittest.TestCase): + copy_if_condition = "exists" + + def test_copy_file_if_same_fs(self): + src_fs = open_fs("mem://") + src_fs.makedir("foo2").touch("exists") + src_fs.makedir("foo1").touch("test1.txt") + self.assertFalse( + fs.copy.copy_file_if( + src_fs, + "foo1/test1.txt", + src_fs, + "foo2/test1.txt.copy", + self.copy_if_condition, + ) + ) + self.assertTrue( + fs.copy.copy_file_if( + src_fs, "foo1/test1.txt", src_fs, "foo2/exists", self.copy_if_condition + ) + ) + self.assertFalse(src_fs.exists("foo2/test1.txt.copy")) + + def test_copy_file_if_dst_doesnt_exists(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + dst_dir = _create_sandbox_dir() + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertFalse(copied) + self.assertFalse(dst_fs.exists("/file1.txt")) finally: shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_fs_if_newer_dont_copy_when_dst_exists(self): + def test_copy_file_if_dst_exists(self): try: - # src is older than dst => no copy should be necessary - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) - - dst_dir = self._create_sandbox_dir() - dst_file1 = self._touch(dst_dir, "file1.txt") - self._write_file(dst_file1) - # ensure dst file is newer than src, changing its modification time - self._delay_file_utime(dst_file1, delta_sec=60) + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertTrue(copied) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_fs_if(self): + try: + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + src_file2 = _touch(src_dir, "file2.txt") + _write_file(src_file1) + _write_file(src_file2) src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) @@ -301,10 +707,13 @@ def test_copy_fs_if_newer_dont_copy_when_dst_exists(self): def on_copy(src_fs, src_path, dst_fs, dst_path): copied.append(dst_path) - fs.copy.copy_fs_if_newer(src_fs, dst_fs, on_copy=on_copy) + fs.copy.copy_fs_if( + src_fs, dst_fs, on_copy=on_copy, condition=self.copy_if_condition + ) - self.assertEqual(copied, []) + self.assertEqual(copied, ["/file1.txt"]) self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertFalse(dst_fs.exists("/file2.txt")) src_fs.close() dst_fs.close() @@ -313,48 +722,249 @@ def on_copy(src_fs, src_path, dst_fs, dst_path): shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_dir_if_newer_one_dst_doesnt_exist(self): + def test_copy_dir_if(self): try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "file1.txt") - self._write_file(src_file1) + src_file2 = _touch(src_dir, os.path.join("one_level_down", "file2.txt")) + _write_file(src_file2) - src_file2 = self._touch(src_dir, "one_level_down" + os.sep + "file2.txt") - self._write_file(src_file2) + dst_dir = _create_sandbox_dir() + mkdirp(os.path.join(dst_dir, "target_dir")) + dst_file1 = _touch(dst_dir, os.path.join("target_dir", "file1.txt")) + _write_file(dst_file1) - dst_dir = self._create_sandbox_dir() - dst_file1 = self._touch(dst_dir, "file1.txt") - self._write_file(dst_file1) - # ensure dst file is newer than src, changing its modification time - self._delay_file_utime(dst_file1, delta_sec=60) + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, + "/", + dst_fs, + "/target_dir/", + on_copy=on_copy, + condition=self.copy_if_condition, + ) + + self.assertEqual(copied, ["/target_dir/file1.txt"]) + self.assertFalse(dst_fs.exists("/target_dir/one_level_down/file2.txt")) + + src_fs.close() + dst_fs.close() + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_dir_if_same_fs(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "src" + os.sep + "file1.txt") + _write_file(src_file1) + + _create_sandbox_dir(home=src_dir) + + src_fs = open_fs("osfs://" + src_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, "/src", src_fs, "/dst", on_copy=on_copy, condition="newer" + ) + + self.assertEqual(copied, ["/dst/file1.txt"]) + self.assertTrue(src_fs.exists("/dst/file1.txt")) + + src_fs.close() + + finally: + shutil.rmtree(src_dir) + + def test_copy_dir_if_multiple_files(self): + try: + src_dir = _create_sandbox_dir() + src_fs = open_fs("osfs://" + src_dir) + src_fs.makedirs("foo/bar") + src_fs.makedirs("foo/empty") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + + dst_dir = _create_sandbox_dir() + dst_fs = open_fs("osfs://" + dst_dir) + + fs.copy.copy_dir_if(src_fs, "/foo", dst_fs, "/", condition="newer") + + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isdir("empty")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + +class TestCopyIfNotExists(unittest.TestCase): + copy_if_condition = "not_exists" + + def test_copy_file_if_same_fs(self): + src_fs = open_fs("mem://") + src_fs.makedir("foo2").touch("exists") + src_fs.makedir("foo1").touch("test1.txt") + self.assertTrue( + fs.copy.copy_file_if( + src_fs, + "foo1/test1.txt", + src_fs, + "foo2/test1.txt.copy", + self.copy_if_condition, + ) + ) + self.assertFalse( + fs.copy.copy_file_if( + src_fs, "foo1/test1.txt", src_fs, "foo2/exists", self.copy_if_condition + ) + ) + self.assertTrue(src_fs.exists("foo2/test1.txt.copy")) + + def test_copy_file_if_dst_doesnt_exists(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() src_fs = open_fs("osfs://" + src_dir) dst_fs = open_fs("osfs://" + dst_dir) + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertTrue(copied) + self.assertTrue(dst_fs.exists("/file1.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_file_if_dst_exists(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) + + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + + copied = fs.copy.copy_file_if( + src_fs, "/file1.txt", dst_fs, "/file1.txt", self.copy_if_condition + ) + + self.assertFalse(copied) + self.assertTrue(dst_fs.exists("/file1.txt")) + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_fs_if(self): + try: + dst_dir = _create_sandbox_dir() + dst_file1 = _touch(dst_dir, "file1.txt") + _write_file(dst_file1) + + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + src_file2 = _touch(src_dir, "file2.txt") + _write_file(src_file1) + _write_file(src_file2) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + self.assertTrue(dst_fs.exists("/file1.txt")) + copied = [] def on_copy(src_fs, src_path, dst_fs, dst_path): copied.append(dst_path) - fs.copy.copy_dir_if_newer(src_fs, "/", dst_fs, "/", on_copy=on_copy) + fs.copy.copy_fs_if( + src_fs, dst_fs, on_copy=on_copy, condition=self.copy_if_condition + ) - self.assertEqual(copied, ["/one_level_down/file2.txt"]) - self.assertTrue(dst_fs.exists("/one_level_down/file2.txt")) + self.assertEqual(copied, ["/file2.txt"]) + self.assertTrue(dst_fs.exists("/file1.txt")) + self.assertTrue(dst_fs.exists("/file2.txt")) src_fs.close() dst_fs.close() + finally: shutil.rmtree(src_dir) shutil.rmtree(dst_dir) - def test_copy_dir_if_newer_same_fs(self): + def test_copy_dir_if(self): try: - src_dir = self._create_sandbox_dir() - src_file1 = self._touch(src_dir, "src" + os.sep + "file1.txt") - self._write_file(src_file1) + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "file1.txt") + _write_file(src_file1) - self._create_sandbox_dir(home=src_dir) + src_file2 = _touch(src_dir, os.path.join("one_level_down", "file2.txt")) + _write_file(src_file2) + + dst_dir = _create_sandbox_dir() + mkdirp(os.path.join(dst_dir, "target_dir")) + dst_file1 = _touch(dst_dir, os.path.join("target_dir", "file1.txt")) + _write_file(dst_file1) + + src_fs = open_fs("osfs://" + src_dir) + dst_fs = open_fs("osfs://" + dst_dir) + + copied = [] + + def on_copy(src_fs, src_path, dst_fs, dst_path): + copied.append(dst_path) + + fs.copy.copy_dir_if( + src_fs, + "/", + dst_fs, + "/target_dir/", + on_copy=on_copy, + condition=self.copy_if_condition, + ) + + self.assertEqual(copied, ["/target_dir/one_level_down/file2.txt"]) + self.assertTrue(dst_fs.exists("/target_dir/file1.txt")) + self.assertTrue(dst_fs.exists("/target_dir/one_level_down/file2.txt")) + + src_fs.close() + dst_fs.close() + finally: + shutil.rmtree(src_dir) + shutil.rmtree(dst_dir) + + def test_copy_dir_if_same_fs(self): + try: + src_dir = _create_sandbox_dir() + src_file1 = _touch(src_dir, "src" + os.sep + "file1.txt") + _write_file(src_file1) + + _create_sandbox_dir(home=src_dir) src_fs = open_fs("osfs://" + src_dir) @@ -363,7 +973,9 @@ def test_copy_dir_if_newer_same_fs(self): def on_copy(src_fs, src_path, dst_fs, dst_path): copied.append(dst_path) - fs.copy.copy_dir_if_newer(src_fs, "/src", src_fs, "/dst", on_copy=on_copy) + fs.copy.copy_dir_if( + src_fs, "/src", src_fs, "/dst", on_copy=on_copy, condition="newer" + ) self.assertEqual(copied, ["/dst/file1.txt"]) self.assertTrue(src_fs.exists("/dst/file1.txt")) @@ -373,19 +985,19 @@ def on_copy(src_fs, src_path, dst_fs, dst_path): finally: shutil.rmtree(src_dir) - def test_copy_dir_if_newer_multiple_files(self): + def test_copy_dir_if_multiple_files(self): try: - src_dir = self._create_sandbox_dir() + src_dir = _create_sandbox_dir() src_fs = open_fs("osfs://" + src_dir) src_fs.makedirs("foo/bar") src_fs.makedirs("foo/empty") src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") - dst_dir = self._create_sandbox_dir() + dst_dir = _create_sandbox_dir() dst_fs = open_fs("osfs://" + dst_dir) - fs.copy.copy_dir_if_newer(src_fs, "/foo", dst_fs, "/") + fs.copy.copy_dir_if(src_fs, "/foo", dst_fs, "/", condition="newer") self.assertTrue(dst_fs.isdir("bar")) self.assertTrue(dst_fs.isdir("empty")) From d3fb588558ff0e9c16e6245a87d6872b244c12b9 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 11:35:26 +0100 Subject: [PATCH 116/309] Updated CHANGELOG.md. --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31cb320c..e7b0419b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added FTP over TLS (FTPS) support to FTPFS. Closes [#437](https://github.com/PyFilesystem/pyfilesystem2/issues/437), [#449](https://github.com/PyFilesystem/pyfilesystem2/pull/449). +- Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. + Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458). ### Changed @@ -31,6 +33,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Make `FTPFile`, `MemoryFile` and `RawWrapper` accept [`array.array`](https://docs.python.org/3/library/array.html) arguments for the `write` and `writelines` methods, as expected by their base class [`io.RawIOBase`](https://docs.python.org/3/library/io.html#io.RawIOBase). - Various documentation issues, including `MemoryFS` docstring not rendering properly. +- Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future. ## [2.4.12] - 2021-01-14 From 1327a6386412ab2d9f0fa1082d2ea1a126daeb04 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 17:56:39 +0100 Subject: [PATCH 117/309] Added `preserve_time` parameter to all copy/move/mirror functions. If `preserve_time` is set to True, the call tries to preserve the atime, ctime, and mtime timestamps on the file after the copy. The value is currently not actually used; places in code that have to be adapted are marked with `TODO(preserve_time)`. --- fs/_bulk.py | 9 ++++--- fs/base.py | 35 ++++++++++++++++++++------ fs/copy.py | 69 ++++++++++++++++++++++++++++++++++++++++++++++------ fs/mirror.py | 15 +++++++++--- fs/move.py | 37 ++++++++++++++++++++++++---- fs/osfs.py | 10 +++++--- fs/wrap.py | 4 +-- fs/wrapfs.py | 12 ++++----- 8 files changed, 153 insertions(+), 38 deletions(-) diff --git a/fs/_bulk.py b/fs/_bulk.py index 9b0b8b79..63cecc01 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -124,13 +124,16 @@ def __exit__( if traceback is None and self.errors: raise BulkCopyFailed(self.errors) - def copy(self, src_fs, src_path, dst_fs, dst_path): - # type: (FS, Text, FS, Text) -> None + def copy(self, src_fs, src_path, dst_fs, dst_path, preserve_time=False): + # type: (FS, Text, FS, Text, bool) -> None """Copy a file from one fs to another.""" if self.queue is None: # This should be the most performant for a single-thread - copy_file_internal(src_fs, src_path, dst_fs, dst_path) + copy_file_internal( + src_fs, src_path, dst_fs, dst_path, preserve_time=preserve_time + ) else: + # TODO(preserve_time) src_file = src_fs.openbin(src_path, "r") try: dst_file = dst_fs.openbin(dst_path, "w") diff --git a/fs/base.py b/fs/base.py index 7a81c5cb..03d4dac8 100644 --- a/fs/base.py +++ b/fs/base.py @@ -387,8 +387,14 @@ def close(self): """ self._closed = True - def copy(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def copy( + self, + src_path, # type: Text + dst_path, # type: Text + overwrite=False, # type: bool + preserve_time=False, # type: bool + ): + # type: (...) -> None """Copy file contents from ``src_path`` to ``dst_path``. Arguments: @@ -396,6 +402,8 @@ def copy(self, src_path, dst_path, overwrite=False): dst_path (str): Path to destination file. overwrite (bool): If `True`, overwrite the destination file if it exists (defaults to `False`). + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). Raises: fs.errors.DestinationExists: If ``dst_path`` exists, @@ -404,6 +412,7 @@ def copy(self, src_path, dst_path, overwrite=False): ``dst_path`` does not exist. """ + # TODO(preserve_time) with self._lock: if not overwrite and self.exists(dst_path): raise errors.DestinationExists(dst_path) @@ -411,8 +420,14 @@ def copy(self, src_path, dst_path, overwrite=False): # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - def copydir(self, src_path, dst_path, create=False): - # type: (Text, Text, bool) -> None + def copydir( + self, + src_path, # type: Text + dst_path, # type: Text + create=False, # type: bool + preserve_time=False, # type: bool + ): + # type: (...) -> None """Copy the contents of ``src_path`` to ``dst_path``. Arguments: @@ -420,6 +435,8 @@ def copydir(self, src_path, dst_path, create=False): dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created if it doesn't exist already (defaults to `False`). + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). Raises: fs.errors.ResourceNotFound: If the ``dst_path`` @@ -431,7 +448,7 @@ def copydir(self, src_path, dst_path, create=False): raise errors.ResourceNotFound(dst_path) if not self.getinfo(src_path).is_dir: raise errors.DirectoryExpected(src_path) - copy.copy_dir(self, src_path, self, dst_path) + copy.copy_dir(self, src_path, self, dst_path, preserve_time=preserve_time) def create(self, path, wipe=False): # type: (Text, bool) -> bool @@ -1004,8 +1021,8 @@ def lock(self): """ return self._lock - def movedir(self, src_path, dst_path, create=False): - # type: (Text, Text, bool) -> None + def movedir(self, src_path, dst_path, create=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None """Move directory ``src_path`` to ``dst_path``. Arguments: @@ -1013,6 +1030,8 @@ def movedir(self, src_path, dst_path, create=False): dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created if it doesn't exist already (defaults to `False`). + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). Raises: fs.errors.ResourceNotFound: if ``dst_path`` does not exist, @@ -1022,7 +1041,7 @@ def movedir(self, src_path, dst_path, create=False): with self._lock: if not create and not self.exists(dst_path): raise errors.ResourceNotFound(dst_path) - move.move_dir(self, src_path, self, dst_path) + move.move_dir(self, src_path, self, dst_path, preserve_time=preserve_time) def makedirs( self, diff --git a/fs/copy.py b/fs/copy.py index 6cd34392..6f9fc701 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -24,6 +24,7 @@ def copy_fs( walker=None, # type: Optional[Walker] on_copy=None, # type: Optional[_OnCopy] workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Copy the contents of one filesystem to another. @@ -39,6 +40,8 @@ def copy_fs( dst_path)``. workers (int): Use `worker` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ return copy_dir( @@ -52,6 +55,7 @@ def copy_fs_if_newer( walker=None, # type: Optional[Walker] on_copy=None, # type: Optional[_OnCopy] workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Copy the contents of one filesystem to another, checking times. @@ -72,10 +76,19 @@ def copy_fs_if_newer( dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ return copy_dir_if_newer( - src_fs, "/", dst_fs, "/", walker=walker, on_copy=on_copy, workers=workers + src_fs, + "/", + dst_fs, + "/", + walker=walker, + on_copy=on_copy, + workers=workers, + preserve_time=preserve_time, ) @@ -113,6 +126,7 @@ def copy_file( src_path, # type: Text dst_fs, # type: Union[FS, Text] dst_path, # type: Text + preserve_time=False, # type: bool ): # type: (...) -> None """Copy a file from one filesystem to another. @@ -124,6 +138,8 @@ def copy_file( src_path (str): Path to a file on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). """ with manage_fs(src_fs, writeable=False) as _src_fs: @@ -131,8 +147,11 @@ def copy_file( if _src_fs is _dst_fs: # Same filesystem, so we can do a potentially optimized # copy - _src_fs.copy(src_path, dst_path, overwrite=True) + _src_fs.copy( + src_path, dst_path, overwrite=True, preserve_time=preserve_time + ) else: + # TODO(preserve_time) # Standard copy with _src_fs.lock(), _dst_fs.lock(): if _dst_fs.hassyspath(dst_path): @@ -148,6 +167,7 @@ def copy_file_internal( src_path, # type: Text dst_fs, # type: FS dst_path, # type: Text + preserve_time=False, # type: bool ): # type: (...) -> None """Copy a file at low level, without calling `manage_fs` or locking. @@ -162,12 +182,15 @@ def copy_file_internal( src_path (str): Path to a file on the source filesystem. dst_fs (FS): Destination filesystem. dst_path (str): Path to a file on the destination filesystem. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). """ if src_fs is dst_fs: # Same filesystem, so we can do a potentially optimized # copy - src_fs.copy(src_path, dst_path, overwrite=True) + src_fs.copy(src_path, dst_path, overwrite=True, preserve_time=preserve_time) + # TODO(preserve_time) elif dst_fs.hassyspath(dst_path): with dst_fs.openbin(dst_path, "w") as write_file: src_fs.download(src_path, write_file) @@ -181,6 +204,7 @@ def copy_file_if_newer( src_path, # type: Text dst_fs, # type: Union[FS, Text] dst_path, # type: Text + preserve_time=False, # type: bool ): # type: (...) -> bool """Copy a file from one filesystem to another, checking times. @@ -196,6 +220,8 @@ def copy_file_if_newer( src_path (str): Path to a file on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). Returns: bool: `True` if the file copy was executed, `False` otherwise. @@ -207,7 +233,9 @@ def copy_file_if_newer( # Same filesystem, so we can do a potentially optimized # copy if _source_is_newer(_src_fs, src_path, _dst_fs, dst_path): - _src_fs.copy(src_path, dst_path, overwrite=True) + _src_fs.copy( + src_path, dst_path, overwrite=True, preserve_time=preserve_time + ) return True else: return False @@ -215,7 +243,13 @@ def copy_file_if_newer( # Standard copy with _src_fs.lock(), _dst_fs.lock(): if _source_is_newer(_src_fs, src_path, _dst_fs, dst_path): - copy_file_internal(_src_fs, src_path, _dst_fs, dst_path) + copy_file_internal( + _src_fs, + src_path, + _dst_fs, + dst_path, + preserve_time=preserve_time, + ) return True else: return False @@ -225,6 +259,7 @@ def copy_structure( src_fs, # type: Union[FS, Text] dst_fs, # type: Union[FS, Text] walker=None, # type: Optional[Walker] + preserve_time=False, # type: bool ): # type: (...) -> None """Copy directories (but not files) from ``src_fs`` to ``dst_fs``. @@ -235,6 +270,8 @@ def copy_structure( walker (~fs.walk.Walker, optional): A walker object that will be used to scan for files in ``src_fs``. Set this if you only want to consider a sub-set of the resources in ``src_fs``. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resource (defaults to `False`). """ walker = walker or Walker() @@ -253,6 +290,7 @@ def copy_dir( walker=None, # type: Optional[Walker] on_copy=None, # type: Optional[_OnCopy] workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Copy a directory from one filesystem to another. @@ -270,6 +308,8 @@ def copy_dir( ``(src_fs, src_path, dst_fs, dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ on_copy = on_copy or (lambda *args: None) @@ -297,7 +337,13 @@ def dst(): for info in files: src_path = info.make_path(dir_path) dst_path = info.make_path(copy_path) - copier.copy(_src_fs, src_path, _dst_fs, dst_path) + copier.copy( + _src_fs, + src_path, + _dst_fs, + dst_path, + preserve_time=preserve_time, + ) on_copy(_src_fs, src_path, _dst_fs, dst_path) @@ -309,6 +355,7 @@ def copy_dir_if_newer( walker=None, # type: Optional[Walker] on_copy=None, # type: Optional[_OnCopy] workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Copy a directory from one filesystem to another, checking times. @@ -331,6 +378,8 @@ def copy_dir_if_newer( ``(src_fs, src_path, dst_fs, dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ on_copy = on_copy or (lambda *args: None) @@ -381,5 +430,11 @@ def dst(): ) if do_copy: - copier.copy(_src_fs, dir_path, _dst_fs, copy_path) + copier.copy( + _src_fs, + dir_path, + _dst_fs, + copy_path, + preserve_time=preserve_time, + ) on_copy(_src_fs, dir_path, _dst_fs, copy_path) diff --git a/fs/mirror.py b/fs/mirror.py index 6b989e63..6123aed2 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -57,6 +57,7 @@ def mirror( walker=None, # type: Optional[Walker] copy_if_newer=True, # type: bool workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Mirror files / directories from one filesystem to another. @@ -73,6 +74,8 @@ def mirror( workers (int): Number of worker threads used (0 for single threaded). Set to a relatively low number for network filesystems, 4 would be a good start. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ @@ -92,13 +95,19 @@ def dst(): walker=walker, copy_if_newer=copy_if_newer, copy_file=copier.copy, + preserve_time=preserve_time, ) def _mirror( - src_fs, dst_fs, walker=None, copy_if_newer=True, copy_file=copy_file_internal + src_fs, # type: FS + dst_fs, # type: FS + walker=None, # type: Optional[Walker] + copy_if_newer=True, # type: bool + copy_file=copy_file_internal, # type: Callable[[FS, str, FS, str, bool], None] + preserve_time=False, # type: bool ): - # type: (FS, FS, Optional[Walker], bool, Callable[[FS, str, FS, str], None]) -> None + # type: (...) -> None walker = walker or Walker() walk = walker.walk(src_fs, namespaces=["details"]) for path, dirs, files in walk: @@ -122,7 +131,7 @@ def _mirror( # Compare file info if copy_if_newer and not _compare(_file, dst_file): continue - copy_file(src_fs, _path, dst_fs, _path) + copy_file(src_fs, _path, dst_fs, _path, preserve_time) # Make directories for _dir in dirs: diff --git a/fs/move.py b/fs/move.py index 1d8e26c1..da941479 100644 --- a/fs/move.py +++ b/fs/move.py @@ -15,8 +15,13 @@ from typing import Text, Union -def move_fs(src_fs, dst_fs, workers=0): - # type: (Union[Text, FS], Union[Text, FS], int) -> None +def move_fs( + src_fs, # type: Union[Text, FS] + dst_fs, # type:Union[Text, FS] + workers=0, # type: int + preserve_time=False, # type: bool +): + # type: (...) -> None """Move the contents of a filesystem to another filesystem. Arguments: @@ -24,9 +29,11 @@ def move_fs(src_fs, dst_fs, workers=0): dst_fs (FS or str): Destination filesystem (instance or URL). workers (int): Use `worker` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ - move_dir(src_fs, "/", dst_fs, "/", workers=workers) + move_dir(src_fs, "/", dst_fs, "/", workers=workers, preserve_time=preserve_time) def move_file( @@ -34,6 +41,7 @@ def move_file( src_path, # type: Text dst_fs, # type: Union[Text, FS] dst_path, # type: Text + preserve_time=False, # type: bool ): # type: (...) -> None """Move a file from one filesystem to another. @@ -43,8 +51,11 @@ def move_file( src_path (str): Path to a file on ``src_fs``. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on ``dst_fs``. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ + # TODO(preserve_time) with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: if _src_fs is _dst_fs: @@ -53,7 +64,13 @@ def move_file( else: # Standard copy and delete with _src_fs.lock(), _dst_fs.lock(): - copy_file(_src_fs, src_path, _dst_fs, dst_path) + copy_file( + _src_fs, + src_path, + _dst_fs, + dst_path, + preserve_time=preserve_time, + ) _src_fs.remove(src_path) @@ -63,6 +80,7 @@ def move_dir( dst_fs, # type: Union[Text, FS] dst_path, # type: Text workers=0, # type: int + preserve_time=False, # type: bool ): # type: (...) -> None """Move a directory from one filesystem to another. @@ -74,6 +92,8 @@ def move_dir( dst_path (str): Path to a directory on ``dst_fs``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). """ @@ -86,5 +106,12 @@ def dst(): with src() as _src_fs, dst() as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): _dst_fs.makedir(dst_path, recreate=True) - copy_dir(src_fs, src_path, dst_fs, dst_path, workers=workers) + copy_dir( + src_fs, + src_path, + dst_fs, + dst_path, + workers=workers, + preserve_time=preserve_time, + ) _src_fs.removetree(src_path) diff --git a/fs/osfs.py b/fs/osfs.py index 3b35541c..8c071c3d 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -431,8 +431,9 @@ def _check_copy(self, src_path, dst_path, overwrite=False): if hasattr(errno, "ENOTSUP"): _sendfile_error_codes.add(errno.ENOTSUP) - def copy(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None + # TODO(preserve_time) with self._lock: # validate and canonicalise paths _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) @@ -461,8 +462,9 @@ def copy(self, src_path, dst_path, overwrite=False): else: - def copy(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None + # TODO(preserve_time) with self._lock: _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path)) diff --git a/fs/wrap.py b/fs/wrap.py index 8685e9f6..9ceae351 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -225,8 +225,8 @@ def settimes(self, path, accessed=None, modified=None): self.check() raise ResourceReadOnly(path) - def copy(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None self.check() raise ResourceReadOnly(dst_path) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index e40a7a83..7be9c235 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -256,17 +256,17 @@ def touch(self, path): with unwrap_errors(path): _fs.touch(_path) - def copy(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): if not overwrite and dst_fs.exists(_dst_path): raise errors.DestinationExists(_dst_path) - copy_file(src_fs, _src_path, dst_fs, _dst_path) + copy_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) - def copydir(self, src_path, dst_path, create=False): - # type: (Text, Text, bool) -> None + def copydir(self, src_path, dst_path, create=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): @@ -274,7 +274,7 @@ def copydir(self, src_path, dst_path, create=False): raise errors.ResourceNotFound(dst_path) if not src_fs.getinfo(_src_path).is_dir: raise errors.DirectoryExpected(src_path) - copy_dir(src_fs, _src_path, dst_fs, _dst_path) + copy_dir(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) def create(self, path, wipe=False): # type: (Text, bool) -> bool From ec39f9d37b457176c66f015f493196ab02b1222f Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 19:05:57 +0100 Subject: [PATCH 118/309] Added unit tests for the `preserve_time` parameter. At this point in time, all the new tests are failing right now; as they should, because the functionality of `preserve_time` has not been implemented yet. --- setup.cfg | 1 + tests/requirements.txt | 2 ++ tests/test_copy.py | 71 +++++++++++++++++++++++++++++++----------- tests/test_memoryfs.py | 16 ++++++++++ tests/test_mirror.py | 53 +++++++++++++++---------------- tests/test_move.py | 36 ++++++++++++++++++++- tests/test_opener.py | 16 ++++++++-- tests/test_osfs.py | 20 ++++++++++++ 8 files changed, 167 insertions(+), 48 deletions(-) diff --git a/setup.cfg b/setup.cfg index b90ebe47..f667b1a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -133,6 +133,7 @@ sitepackages = false skip_missing_interpreters = true requires = setuptools >=38.3.0 + parameterized ~=0.8 [testenv] commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests diff --git a/tests/requirements.txt b/tests/requirements.txt index 9e7ece32..b7ff3ce4 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -14,3 +14,5 @@ pysendfile ~=2.0 ; python_version <= "3.3" # mock v4+ doesn't support Python 2.7 anymore mock ~=3.0 ; python_version < "3.3" +# parametrized to prevent code duplication in tests. +parameterized ~=0.8 \ No newline at end of file diff --git a/tests/test_copy.py b/tests/test_copy.py index 63e550e9..77c7f756 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -8,25 +8,44 @@ import shutil import calendar +from parameterized import parameterized + import fs.copy from fs import open_fs class TestCopy(unittest.TestCase): - def test_copy_fs(self): - for workers in (0, 1, 2, 4): - src_fs = open_fs("mem://") - src_fs.makedirs("foo/bar") - src_fs.makedirs("foo/empty") - src_fs.touch("test.txt") - src_fs.touch("foo/bar/baz.txt") + @parameterized.expand([(0,), (1,), (2,), (4,)]) + def test_copy_fs(self, workers): + namespaces = ("details", "accessed", "metadata_changed", "modified") - dst_fs = open_fs("mem://") - fs.copy.copy_fs(src_fs, dst_fs, workers=workers) + src_fs = open_fs("mem://") + src_fs.makedirs("foo/bar") + src_fs.makedirs("foo/empty") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + src_file1_info = src_fs.getinfo("test.txt", namespaces) + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) - self.assertTrue(dst_fs.isdir("foo/empty")) - self.assertTrue(dst_fs.isdir("foo/bar")) - self.assertTrue(dst_fs.isfile("test.txt")) + dst_fs = open_fs("mem://") + fs.copy.copy_fs(src_fs, dst_fs, workers=workers, preserve_time=True) + + self.assertTrue(dst_fs.isdir("foo/empty")) + self.assertTrue(dst_fs.isdir("foo/bar")) + self.assertTrue(dst_fs.isfile("test.txt")) + + dst_file1_info = dst_fs.getinfo("test.txt", namespaces) + dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) + self.assertEqual(dst_file1_info.modified, src_file1_info.modified) + self.assertEqual(dst_file1_info.accessed, src_file1_info.accessed) + self.assertEqual( + dst_file1_info.metadata_changed, src_file1_info.metadata_changed + ) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) + self.assertEqual( + dst_file2_info.metadata_changed, src_file2_info.metadata_changed + ) def test_copy_value_error(self): src_fs = open_fs("mem://") @@ -34,18 +53,32 @@ def test_copy_value_error(self): with self.assertRaises(ValueError): fs.copy.copy_fs(src_fs, dst_fs, workers=-1) - def test_copy_dir(self): + @parameterized.expand([(0,), (1,), (2,), (4,)]) + def test_copy_dir(self, workers): + namespaces = ("details", "accessed", "metadata_changed", "modified") + src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") src_fs.makedirs("foo/empty") src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") - for workers in (0, 1, 2, 4): - with open_fs("mem://") as dst_fs: - fs.copy.copy_dir(src_fs, "/foo", dst_fs, "/", workers=workers) - self.assertTrue(dst_fs.isdir("bar")) - self.assertTrue(dst_fs.isdir("empty")) - self.assertTrue(dst_fs.isfile("bar/baz.txt")) + src_file1_info = src_fs.getinfo("test.txt", namespaces) + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) + + with open_fs("mem://") as dst_fs: + fs.copy.copy_dir( + src_fs, "/foo", dst_fs, "/", workers=workers, preserve_time=True + ) + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isdir("empty")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) + self.assertEqual( + dst_file2_info.metadata_changed, src_file2_info.metadata_changed + ) def test_copy_large(self): data1 = b"foo" * 512 * 1024 diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 0b26a576..59502cbe 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -66,3 +66,19 @@ def test_close_mem_free(self): "Memory usage increased after closing the file system; diff is %0.2f KiB." % (diff_close.size_diff / 1024.0), ) + + def test_copy_preserve_time(self): + self.fs.makedir("foo") + self.fs.makedir("bar") + self.fs.touch("foo/file.txt") + + namespaces = ("details", "accessed", "metadata_changed", "modified") + src_info = self.fs.getinfo("foo/file.txt", namespaces) + + self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) + self.assertTrue(self.fs.exists("bar/file.txt")) + + dst_info = self.fs.getinfo("bar/file.txt", namespaces) + self.assertEqual(dst_info.modified, src_info.modified) + self.assertEqual(dst_info.accessed, src_info.accessed) + self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) diff --git a/tests/test_mirror.py b/tests/test_mirror.py index a0e8ac53..f204178f 100644 --- a/tests/test_mirror.py +++ b/tests/test_mirror.py @@ -2,15 +2,17 @@ import unittest +from parameterized import parameterized_class + from fs.mirror import mirror from fs import open_fs +@parameterized_class(("WORKERS",), [(0,), (1,), (2,), (4,)]) class TestMirror(unittest.TestCase): - WORKERS = 0 # Single threaded - def _contents(self, fs): """Extract an FS in to a simple data structure.""" + namespaces = ("details", "accessed", "metadata_changed", "modified") contents = [] for path, dirs, files in fs.walk(): for info in dirs: @@ -18,7 +20,18 @@ def _contents(self, fs): contents.append((_path, "dir", b"")) for info in files: _path = info.make_path(path) - contents.append((_path, "file", fs.readbytes(_path))) + _bytes = fs.readbytes(_path) + _info = fs.getinfo(_path, namespaces) + contents.append( + ( + _path, + "file", + _bytes, + _info.modified, + _info.accessed, + _info.metadata_changed, + ) + ) return sorted(contents) def assert_compare_fs(self, fs1, fs2): @@ -28,14 +41,14 @@ def assert_compare_fs(self, fs1, fs2): def test_empty_mirror(self): m1 = open_fs("mem://") m2 = open_fs("mem://") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_one_file(self): m1 = open_fs("mem://") m1.writetext("foo", "hello") m2 = open_fs("mem://") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_one_file_one_dir(self): @@ -43,7 +56,7 @@ def test_mirror_one_file_one_dir(self): m1.writetext("foo", "hello") m1.makedir("bar") m2 = open_fs("mem://") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_delete_replace(self): @@ -51,13 +64,13 @@ def test_mirror_delete_replace(self): m1.writetext("foo", "hello") m1.makedir("bar") m2 = open_fs("mem://") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) m2.remove("foo") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) m2.removedir("bar") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_extra_dir(self): @@ -66,7 +79,7 @@ def test_mirror_extra_dir(self): m1.makedir("bar") m2 = open_fs("mem://") m2.makedir("baz") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_extra_file(self): @@ -76,7 +89,7 @@ def test_mirror_extra_file(self): m2 = open_fs("mem://") m2.makedir("baz") m2.touch("egg") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_wrong_type(self): @@ -86,7 +99,7 @@ def test_mirror_wrong_type(self): m2 = open_fs("mem://") m2.makedir("foo") m2.touch("bar") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) def test_mirror_update(self): @@ -94,20 +107,8 @@ def test_mirror_update(self): m1.writetext("foo", "hello") m1.makedir("bar") m2 = open_fs("mem://") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) m2.appendtext("foo", " world!") - mirror(m1, m2, workers=self.WORKERS) + mirror(m1, m2, workers=self.WORKERS, preserve_time=True) self.assert_compare_fs(m1, m2) - - -class TestMirrorWorkers1(TestMirror): - WORKERS = 1 - - -class TestMirrorWorkers2(TestMirror): - WORKERS = 2 - - -class TestMirrorWorkers4(TestMirror): - WORKERS = 4 diff --git a/tests/test_move.py b/tests/test_move.py index d87d2bd6..8bba8857 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -2,29 +2,54 @@ import unittest +from parameterized import parameterized_class + import fs.move from fs import open_fs +@parameterized_class(("preserve_time",), [(True,), (False,)]) class TestMove(unittest.TestCase): def test_move_fs(self): + namespaces = ("details", "accessed", "metadata_changed", "modified") + src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") + src_file1_info = src_fs.getinfo("test.txt", namespaces) + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") fs.move.move_fs(src_fs, dst_fs) self.assertTrue(dst_fs.isdir("foo/bar")) self.assertTrue(dst_fs.isfile("test.txt")) + self.assertTrue(dst_fs.isfile("foo/bar/baz.txt")) self.assertTrue(src_fs.isempty("/")) - def test_copy_dir(self): + if self.preserve_time: + dst_file1_info = dst_fs.getinfo("test.txt", namespaces) + dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) + self.assertEqual(dst_file1_info.modified, src_file1_info.modified) + self.assertEqual(dst_file1_info.accessed, src_file1_info.accessed) + self.assertEqual( + dst_file1_info.metadata_changed, src_file1_info.metadata_changed + ) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) + self.assertEqual( + dst_file2_info.metadata_changed, src_file2_info.metadata_changed + ) + + def test_move_dir(self): + namespaces = ("details", "accessed", "metadata_changed", "modified") + src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") fs.move.move_dir(src_fs, "/foo", dst_fs, "/") @@ -32,3 +57,12 @@ def test_copy_dir(self): self.assertTrue(dst_fs.isdir("bar")) self.assertTrue(dst_fs.isfile("bar/baz.txt")) self.assertFalse(src_fs.exists("foo")) + self.assertTrue(src_fs.isfile("test.txt")) + + if self.preserve_time: + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) + self.assertEqual( + dst_file2_info.metadata_changed, src_file2_info.metadata_changed + ) diff --git a/tests/test_opener.py b/tests/test_opener.py index e7fae983..bc2f5cd7 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -300,14 +300,26 @@ def test_user_data_opener(self, app_dir): def test_open_ftp(self, mock_FTPFS): open_fs("ftp://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( - "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=False + "ftp.example.org", + passwd="bar", + port=21, + user="foo", + proxy=None, + timeout=10, + tls=False, ) @mock.patch("fs.ftpfs.FTPFS") def test_open_ftps(self, mock_FTPFS): open_fs("ftps://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( - "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=True + "ftp.example.org", + passwd="bar", + port=21, + user="foo", + proxy=None, + timeout=10, + tls=True, ) @mock.patch("fs.ftpfs.FTPFS") diff --git a/tests/test_osfs.py b/tests/test_osfs.py index e43635f4..56927367 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -7,6 +7,7 @@ import shutil import tempfile import sys +import time import unittest from fs import osfs, open_fs @@ -87,6 +88,25 @@ def test_expand_vars(self): self.assertIn("TYRIONLANISTER", fs1.getsyspath("/")) self.assertNotIn("TYRIONLANISTER", fs2.getsyspath("/")) + def test_copy_preserve_time(self): + self.fs.makedir("foo") + self.fs.makedir("bar") + now = time.time() - 10000 + if not self.fs.create("foo/file.txt"): + raw_info = {"details": {"accessed": now, "modified": now}} + self.fs.setinfo("foo/file.txt", raw_info) + + namespaces = ("details", "accessed", "metadata_changed", "modified") + src_info = self.fs.getinfo("foo/file.txt", namespaces) + + self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) + self.assertTrue(self.fs.exists("bar/file.txt")) + + dst_info = self.fs.getinfo("bar/file.txt", namespaces) + self.assertEqual(dst_info.modified, src_info.modified) + self.assertEqual(dst_info.accessed, src_info.accessed) + self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) + @unittest.skipUnless(osfs.sendfile, "sendfile not supported") @unittest.skipIf( sys.version_info >= (3, 8), From f016533871b16b621b6b9ba542bea2e35225ea12 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 19:43:40 +0100 Subject: [PATCH 119/309] Added fs.copy.copy_metadata. All logic for the added `preserve_time` parameter is contained in that function, so other pieces of code just have to make a single call. It's still not implemented at this point. --- fs/_bulk.py | 43 +++++++++++++++++++++++-------------------- fs/base.py | 14 +++++++++----- fs/copy.py | 39 +++++++++++++++++++++++++++++++++------ fs/mirror.py | 6 +++--- fs/move.py | 5 +++-- fs/osfs.py | 2 -- fs/wrap.py | 4 ++-- fs/wrapfs.py | 6 +++--- 8 files changed, 76 insertions(+), 43 deletions(-) diff --git a/fs/_bulk.py b/fs/_bulk.py index 63cecc01..ec832945 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -56,20 +56,30 @@ def __call__(self): class _CopyTask(_Task): """A callable that copies from one file another.""" - def __init__(self, src_file, dst_file): - # type: (IO, IO) -> None - self.src_file = src_file - self.dst_file = dst_file + def __init__( + self, + src_fs, # type: FS + src_path, # type: Text + dst_fs, # type: FS + dst_path, # type: Text + preserve_time, # type: bool + ): + # type: (...) -> None + self.src_fs = src_fs + self.src_path = src_path + self.dst_fs = dst_fs + self.dst_path = dst_path + self.preserve_time = preserve_time def __call__(self): # type: () -> None - try: - copy_file_data(self.src_file, self.dst_file, chunk_size=1024 * 1024) - finally: - try: - self.src_file.close() - finally: - self.dst_file.close() + copy_file_internal( + self.src_fs, + self.src_path, + self.dst_fs, + self.dst_path, + preserve_time=self.preserve_time, + ) class Copier(object): @@ -88,7 +98,7 @@ def __init__(self, num_workers=4): def start(self): """Start the workers.""" if self.num_workers: - self.queue = Queue(maxsize=self.num_workers) + self.queue = Queue() self.workers = [_Worker(self) for _ in range(self.num_workers)] for worker in self.workers: worker.start() @@ -133,12 +143,5 @@ def copy(self, src_fs, src_path, dst_fs, dst_path, preserve_time=False): src_fs, src_path, dst_fs, dst_path, preserve_time=preserve_time ) else: - # TODO(preserve_time) - src_file = src_fs.openbin(src_path, "r") - try: - dst_file = dst_fs.openbin(dst_path, "w") - except Exception: - src_file.close() - raise - task = _CopyTask(src_file, dst_file) + task = _CopyTask(src_fs, src_path, dst_fs, dst_path, preserve_time) self.queue.put(task) diff --git a/fs/base.py b/fs/base.py index 03d4dac8..c86a5de6 100644 --- a/fs/base.py +++ b/fs/base.py @@ -22,6 +22,7 @@ import six from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard +from .copy import copy_metadata from .glob import BoundGlobber from .mode import validate_open_mode from .path import abspath, join, normpath @@ -412,13 +413,13 @@ def copy( ``dst_path`` does not exist. """ - # TODO(preserve_time) with self._lock: if not overwrite and self.exists(dst_path): raise errors.DestinationExists(dst_path) with closing(self.open(src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore + copy_metadata(self, src_path, self, dst_path) def copydir( self, @@ -1030,8 +1031,8 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created if it doesn't exist already (defaults to `False`). - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). Raises: fs.errors.ResourceNotFound: if ``dst_path`` does not exist, @@ -1086,8 +1087,8 @@ def makedirs( raise return self.opendir(path) - def move(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def move(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None """Move a file from ``src_path`` to ``dst_path``. Arguments: @@ -1096,6 +1097,8 @@ def move(self, src_path, dst_path, overwrite=False): file will be written to. overwrite (bool): If `True`, destination path will be overwritten if it exists. + preserve_time (bool): If `True`, try to preserve atime, ctime, + and mtime of the resources (defaults to `False`). Raises: fs.errors.FileExpected: If ``src_path`` maps to a @@ -1128,6 +1131,7 @@ def move(self, src_path, dst_path, overwrite=False): # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore self.remove(src_path) + copy_metadata(self, src_path, self, dst_path) def open( self, diff --git a/fs/copy.py b/fs/copy.py index 6f9fc701..8cd2a55f 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -151,7 +151,6 @@ def copy_file( src_path, dst_path, overwrite=True, preserve_time=preserve_time ) else: - # TODO(preserve_time) # Standard copy with _src_fs.lock(), _dst_fs.lock(): if _dst_fs.hassyspath(dst_path): @@ -160,6 +159,7 @@ def copy_file( else: with _src_fs.openbin(src_path) as read_file: _dst_fs.upload(dst_path, read_file) + copy_metadata(_src_fs, src_path, _dst_fs, dst_path) def copy_file_internal( @@ -190,14 +190,17 @@ def copy_file_internal( # Same filesystem, so we can do a potentially optimized # copy src_fs.copy(src_path, dst_path, overwrite=True, preserve_time=preserve_time) - # TODO(preserve_time) - elif dst_fs.hassyspath(dst_path): + return + + if dst_fs.hassyspath(dst_path): with dst_fs.openbin(dst_path, "w") as write_file: src_fs.download(src_path, write_file) else: with src_fs.openbin(src_path) as read_file: dst_fs.upload(dst_path, read_file) + copy_metadata(src_fs, src_path, dst_fs, dst_path) + def copy_file_if_newer( src_fs, # type: Union[FS, Text] @@ -326,9 +329,10 @@ def dst(): from ._bulk import Copier with src() as _src_fs, dst() as _dst_fs: - with _src_fs.lock(), _dst_fs.lock(): - _thread_safe = is_thread_safe(_src_fs, _dst_fs) - with Copier(num_workers=workers if _thread_safe else 0) as copier: + _thread_safe = is_thread_safe(_src_fs, _dst_fs) + copier = Copier(num_workers=workers if _thread_safe else 0) + with copier: + with _src_fs.lock(), _dst_fs.lock(): _dst_fs.makedir(_dst_path, recreate=True) for dir_path, dirs, files in walker.walk(_src_fs, _src_path): copy_path = combine(_dst_path, frombase(_src_path, dir_path)) @@ -345,6 +349,7 @@ def dst(): preserve_time=preserve_time, ) on_copy(_src_fs, src_path, _dst_fs, dst_path) + pass def copy_dir_if_newer( @@ -438,3 +443,25 @@ def dst(): preserve_time=preserve_time, ) on_copy(_src_fs, dir_path, _dst_fs, copy_path) + + +def copy_metadata( + src_fs, # type: Union[FS, Text] + src_path, # type: Text + dst_fs, # type: Union[FS, Text] + dst_path, # type: Text +): + # type: (...) -> None + """Copies metadata from one file to another. + + Arguments: + src_fs (FS or str): Source filesystem (instance or URL). + src_path (str): Path to a directory on the source filesystem. + dst_fs (FS or str): Destination filesystem (instance or URL). + dst_path (str): Path to a directory on the destination filesystem. + + """ + # TODO(preserve_time) + with manage_fs(src_fs, writeable=False) as _src_fs: + with manage_fs(dst_fs, create=True) as _dst_fs: + pass diff --git a/fs/mirror.py b/fs/mirror.py index 6123aed2..1c674cbc 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -86,9 +86,9 @@ def dst(): return manage_fs(dst_fs, create=True) with src() as _src_fs, dst() as _dst_fs: - with _src_fs.lock(), _dst_fs.lock(): - _thread_safe = is_thread_safe(_src_fs, _dst_fs) - with Copier(num_workers=workers if _thread_safe else 0) as copier: + _thread_safe = is_thread_safe(_src_fs, _dst_fs) + with Copier(num_workers=workers if _thread_safe else 0) as copier: + with _src_fs.lock(), _dst_fs.lock(): _mirror( _src_fs, _dst_fs, diff --git a/fs/move.py b/fs/move.py index da941479..56dbcb98 100644 --- a/fs/move.py +++ b/fs/move.py @@ -55,12 +55,13 @@ def move_file( and mtime of the resources (defaults to `False`). """ - # TODO(preserve_time) with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: if _src_fs is _dst_fs: # Same filesystem, may be optimized - _src_fs.move(src_path, dst_path, overwrite=True) + _src_fs.move( + src_path, dst_path, overwrite=True, preserve_time=preserve_time + ) else: # Standard copy and delete with _src_fs.lock(), _dst_fs.lock(): diff --git a/fs/osfs.py b/fs/osfs.py index 8c071c3d..47d612be 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -433,7 +433,6 @@ def _check_copy(self, src_path, dst_path, overwrite=False): def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None - # TODO(preserve_time) with self._lock: # validate and canonicalise paths _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) @@ -464,7 +463,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None - # TODO(preserve_time) with self._lock: _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path)) diff --git a/fs/wrap.py b/fs/wrap.py index 9ceae351..ce7e9d3c 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -181,8 +181,8 @@ def makedir( self.check() raise ResourceReadOnly(path) - def move(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def move(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None self.check() raise ResourceReadOnly(dst_path) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 7be9c235..1419b5aa 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -167,15 +167,15 @@ def makedir( with unwrap_errors(path): return _fs.makedir(_path, permissions=permissions, recreate=recreate) - def move(self, src_path, dst_path, overwrite=False): - # type: (Text, Text, bool) -> None + def move(self, src_path, dst_path, overwrite=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None # A custom move permits a potentially optimized code path src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): if not overwrite and dst_fs.exists(_dst_path): raise errors.DestinationExists(_dst_path) - move_file(src_fs, _src_path, dst_fs, _dst_path) + move_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) def movedir(self, src_path, dst_path, create=False): # type: (Text, Text, bool) -> None From 34d26bce9c261dd8b3f67f1928ced5a56578a8c1 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 20 Mar 2021 20:09:22 +0100 Subject: [PATCH 120/309] Removed preservation of access time in copy/move/mirror functions as changing the access time itself was an access. --- fs/_bulk.py | 3 +-- fs/base.py | 6 +++--- fs/copy.py | 18 ++++++++++++------ fs/wrapfs.py | 4 ++-- tests/test_copy.py | 26 ++++++++++++++++++++++---- tests/test_memoryfs.py | 1 - tests/test_mirror.py | 3 +-- tests/test_move.py | 3 --- 8 files changed, 41 insertions(+), 23 deletions(-) diff --git a/fs/_bulk.py b/fs/_bulk.py index ec832945..7ee1a2c7 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -13,12 +13,11 @@ from .copy import copy_file_internal from .errors import BulkCopyFailed -from .tools import copy_file_data if typing.TYPE_CHECKING: from .base import FS from types import TracebackType - from typing import IO, List, Optional, Text, Type + from typing import List, Optional, Text, Type class _Worker(threading.Thread): diff --git a/fs/base.py b/fs/base.py index c86a5de6..377bba72 100644 --- a/fs/base.py +++ b/fs/base.py @@ -22,7 +22,7 @@ import six from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard -from .copy import copy_metadata +from .copy import copy_cmtime from .glob import BoundGlobber from .mode import validate_open_mode from .path import abspath, join, normpath @@ -419,7 +419,7 @@ def copy( with closing(self.open(src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - copy_metadata(self, src_path, self, dst_path) + copy_cmtime(self, src_path, self, dst_path) def copydir( self, @@ -1130,8 +1130,8 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): with self.open(src_path, "rb") as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore + copy_cmtime(self, src_path, self, dst_path) self.remove(src_path) - copy_metadata(self, src_path, self, dst_path) def open( self, diff --git a/fs/copy.py b/fs/copy.py index 8cd2a55f..5a252891 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -159,7 +159,7 @@ def copy_file( else: with _src_fs.openbin(src_path) as read_file: _dst_fs.upload(dst_path, read_file) - copy_metadata(_src_fs, src_path, _dst_fs, dst_path) + copy_cmtime(_src_fs, src_path, _dst_fs, dst_path) def copy_file_internal( @@ -199,7 +199,7 @@ def copy_file_internal( with src_fs.openbin(src_path) as read_file: dst_fs.upload(dst_path, read_file) - copy_metadata(src_fs, src_path, dst_fs, dst_path) + copy_cmtime(src_fs, src_path, dst_fs, dst_path) def copy_file_if_newer( @@ -445,14 +445,14 @@ def dst(): on_copy(_src_fs, dir_path, _dst_fs, copy_path) -def copy_metadata( +def copy_cmtime( src_fs, # type: Union[FS, Text] src_path, # type: Text dst_fs, # type: Union[FS, Text] dst_path, # type: Text ): # type: (...) -> None - """Copies metadata from one file to another. + """Copy modified time metadata from one file to another. Arguments: src_fs (FS or str): Source filesystem (instance or URL). @@ -461,7 +461,13 @@ def copy_metadata( dst_path (str): Path to a directory on the destination filesystem. """ - # TODO(preserve_time) + namespaces = ("details", "metadata_changed", "modified") with manage_fs(src_fs, writeable=False) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: - pass + src_meta = _src_fs.getinfo(src_path, namespaces) + src_details = src_meta.raw.get("details", {}) + dst_details = {} + for value in ("metadata_changed", "modified"): + if value in src_details: + dst_details[value] = src_details[value] + _dst_fs.setinfo(dst_path, {"details": dst_details}) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 1419b5aa..071878c2 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -177,8 +177,8 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): raise errors.DestinationExists(_dst_path) move_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) - def movedir(self, src_path, dst_path, create=False): - # type: (Text, Text, bool) -> None + def movedir(self, src_path, dst_path, create=False, preserve_time=False): + # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): diff --git a/tests/test_copy.py b/tests/test_copy.py index 77c7f756..f2f279ba 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -37,12 +37,10 @@ def test_copy_fs(self, workers): dst_file1_info = dst_fs.getinfo("test.txt", namespaces) dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) self.assertEqual(dst_file1_info.modified, src_file1_info.modified) - self.assertEqual(dst_file1_info.accessed, src_file1_info.accessed) self.assertEqual( dst_file1_info.metadata_changed, src_file1_info.metadata_changed ) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) self.assertEqual( dst_file2_info.metadata_changed, src_file2_info.metadata_changed ) @@ -53,6 +51,28 @@ def test_copy_value_error(self): with self.assertRaises(ValueError): fs.copy.copy_fs(src_fs, dst_fs, workers=-1) + def test_copy_dir0(self): + namespaces = ("details", "accessed", "metadata_changed", "modified") + + src_fs = open_fs("mem://") + src_fs.makedirs("foo/bar") + src_fs.makedirs("foo/empty") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) + + with open_fs("mem://") as dst_fs: + fs.copy.copy_dir(src_fs, "/foo", dst_fs, "/", workers=0, preserve_time=True) + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isdir("empty")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertEqual( + dst_file2_info.metadata_changed, src_file2_info.metadata_changed + ) + @parameterized.expand([(0,), (1,), (2,), (4,)]) def test_copy_dir(self, workers): namespaces = ("details", "accessed", "metadata_changed", "modified") @@ -62,7 +82,6 @@ def test_copy_dir(self, workers): src_fs.makedirs("foo/empty") src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") - src_file1_info = src_fs.getinfo("test.txt", namespaces) src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) with open_fs("mem://") as dst_fs: @@ -75,7 +94,6 @@ def test_copy_dir(self, workers): dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) self.assertEqual( dst_file2_info.metadata_changed, src_file2_info.metadata_changed ) diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 59502cbe..253cbb1f 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -80,5 +80,4 @@ def test_copy_preserve_time(self): dst_info = self.fs.getinfo("bar/file.txt", namespaces) self.assertEqual(dst_info.modified, src_info.modified) - self.assertEqual(dst_info.accessed, src_info.accessed) self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) diff --git a/tests/test_mirror.py b/tests/test_mirror.py index f204178f..1cce3d59 100644 --- a/tests/test_mirror.py +++ b/tests/test_mirror.py @@ -12,7 +12,7 @@ class TestMirror(unittest.TestCase): def _contents(self, fs): """Extract an FS in to a simple data structure.""" - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "metadata_changed", "modified") contents = [] for path, dirs, files in fs.walk(): for info in dirs: @@ -28,7 +28,6 @@ def _contents(self, fs): "file", _bytes, _info.modified, - _info.accessed, _info.metadata_changed, ) ) diff --git a/tests/test_move.py b/tests/test_move.py index 8bba8857..04a93628 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -32,12 +32,10 @@ def test_move_fs(self): dst_file1_info = dst_fs.getinfo("test.txt", namespaces) dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) self.assertEqual(dst_file1_info.modified, src_file1_info.modified) - self.assertEqual(dst_file1_info.accessed, src_file1_info.accessed) self.assertEqual( dst_file1_info.metadata_changed, src_file1_info.metadata_changed ) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) self.assertEqual( dst_file2_info.metadata_changed, src_file2_info.metadata_changed ) @@ -62,7 +60,6 @@ def test_move_dir(self): if self.preserve_time: dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual(dst_file2_info.accessed, src_file2_info.accessed) self.assertEqual( dst_file2_info.metadata_changed, src_file2_info.metadata_changed ) From 3a7de8fcd6a947d57292d5903084223c5fc1829a Mon Sep 17 00:00:00 2001 From: atollk Date: Mon, 22 Mar 2021 18:06:30 +0100 Subject: [PATCH 121/309] Marked fs.copy.copy_*_if_newer as deprecated. --- fs/copy.py | 67 ++++++------------------------------------------------ 1 file changed, 7 insertions(+), 60 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 3f949cc3..e2e82790 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals import typing +import warnings from .errors import ResourceNotFound from .opener import manage_fs @@ -52,26 +53,8 @@ def copy_fs_if_newer( workers=0, # type: int ): # type: (...) -> None - """Copy the contents of one filesystem to another, checking times. - - If both source and destination files exist, the copy is executed - only if the source file is newer than the destination file. In case - modification times of source or destination files are not available, - copy file is always executed. - - Arguments: - src_fs (FS or str): Source filesystem (URL or instance). - dst_fs (FS or str): Destination filesystem (URL or instance). - walker (~fs.walk.Walker, optional): A walker object that will be - used to scan for files in ``src_fs``. Set this if you only want - to consider a sub-set of the resources in ``src_fs``. - on_copy (callable):A function callback called after a single file copy - is executed. Expected signature is ``(src_fs, src_path, dst_fs, - dst_path)``. - workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for - a single-threaded copy. - - """ + """Deprecated. Use ``copy_fs_if``.""" + warnings.warn(DeprecationWarning("copy_fs_if_newer is deprecated. Use copy_fs_if instead.")) return copy_fs_if(src_fs, dst_fs, "newer", walker, on_copy, workers) @@ -157,24 +140,8 @@ def copy_file_if_newer( dst_path, # type: Text ): # type: (...) -> bool - """Copy a file from one filesystem to another, checking times. - - If the destination exists, and is a file, it will be first truncated. - If both source and destination files exist, the copy is executed only - if the source file is newer than the destination file. In case - modification times of source or destination files are not available, - copy is always executed. - - Arguments: - src_fs (FS or str): Source filesystem (instance or URL). - src_path (str): Path to a file on the source filesystem. - dst_fs (FS or str): Destination filesystem (instance or URL). - dst_path (str): Path to a file on the destination filesystem. - - Returns: - bool: `True` if the file copy was executed, `False` otherwise. - - """ + """Deprecated. Use ``copy_file_if``.""" + warnings.warn(DeprecationWarning("copy_file_if_newer is deprecated. Use copy_file_if instead.")) return copy_file_if(src_fs, src_path, dst_fs, dst_path, "newer") @@ -342,28 +309,8 @@ def copy_dir_if_newer( workers=0, # type: int ): # type: (...) -> None - """Copy a directory from one filesystem to another, checking times. - - If both source and destination files exist, the copy is executed only - if the source file is newer than the destination file. In case - modification times of source or destination files are not available, - copy is always executed. - - Arguments: - src_fs (FS or str): Source filesystem (instance or URL). - src_path (str): Path to a directory on the source filesystem. - dst_fs (FS or str): Destination filesystem (instance or URL). - dst_path (str): Path to a directory on the destination filesystem. - walker (~fs.walk.Walker, optional): A walker object that will be - used to scan for files in ``src_fs``. Set this if you only - want to consider a sub-set of the resources in ``src_fs``. - on_copy (callable, optional): A function callback called after - a single file copy is executed. Expected signature is - ``(src_fs, src_path, dst_fs, dst_path)``. - workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for - a single-threaded copy. - - """ + """Deprecated. Use ``copy_dir_if``.""" + warnings.warn(DeprecationWarning("copy_dir_if_newer is deprecated. Use copy_dir_if instead.")) copy_dir_if(src_fs, src_path, dst_fs, dst_path, "newer", walker, on_copy, workers) From 8cc977ef02c8fc26b1276c9b4ce0b0fedd9f5a99 Mon Sep 17 00:00:00 2001 From: atollk Date: Mon, 22 Mar 2021 18:14:40 +0100 Subject: [PATCH 122/309] Applied changes suggested by code review. --- fs/copy.py | 139 ++++++++++++++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 66 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index e2e82790..55172afd 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -53,8 +53,10 @@ def copy_fs_if_newer( workers=0, # type: int ): # type: (...) -> None - """Deprecated. Use ``copy_fs_if``.""" - warnings.warn(DeprecationWarning("copy_fs_if_newer is deprecated. Use copy_fs_if instead.")) + """Use ``copy_fs_if`` instead.""" + warnings.warn( + DeprecationWarning("copy_fs_if_newer is deprecated. Use copy_fs_if instead.") + ) return copy_fs_if(src_fs, dst_fs, "newer", walker, on_copy, workers) @@ -70,21 +72,22 @@ def copy_fs_if( """Copy the contents of one filesystem to another, depending on a condition. Depending on the value of ``strategy``, certain conditions must be fulfilled - for a file to be copied to ``dst_fs``. - - If ``condition`` has the value ``"newer"``, the last modification time - of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. - - If ``condition`` has the value ``"older"``, the last modification time - of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. - - If ``condition`` has the value ``"exists"``, the source file is only - copied if a file of the same path already exists in ``dst_fs``. + for a file to be copied to ``dst_fs``. The following values + are supported: + + ``"always"`` + The source file is always copied. + ``"newer"`` + The last modification time of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"older"`` + The last modification time of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"exists"`` + The source file is only copied if a file of the same path already exists in ``dst_fs``. + ``"not_exists"`` + The source file is only copied if no file of the same path already exists in ``dst_fs``. - If ``condition`` has the value ``"not_exists"``, the source file is only - copied if no file of the same path already exists in ``dst_fs``. Arguments: src_fs (FS or str): Source filesystem (URL or instance). @@ -140,8 +143,12 @@ def copy_file_if_newer( dst_path, # type: Text ): # type: (...) -> bool - """Deprecated. Use ``copy_file_if``.""" - warnings.warn(DeprecationWarning("copy_file_if_newer is deprecated. Use copy_file_if instead.")) + """Use ``copy_file_if`` instead.""" + warnings.warn( + DeprecationWarning( + "copy_file_if_newer is deprecated. Use copy_file_if instead." + ) + ) return copy_file_if(src_fs, src_path, dst_fs, dst_path, "newer") @@ -156,21 +163,22 @@ def copy_file_if( """Copy a file from one filesystem to another, depending on a condition. Depending on the value of ``strategy``, certain conditions must be fulfilled - for a file to be copied to ``dst_fs``. - - If ``condition`` has the value ``"newer"``, the last modification time - of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. - - If ``condition`` has the value ``"older"``, the last modification time - of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. + for a file to be copied to ``dst_fs``. The following values + are supported: + + ``"always"`` + The source file is always copied. + ``"newer"`` + The last modification time of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"older"`` + The last modification time of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"exists"`` + The source file is only copied if a file of the same path already exists in ``dst_fs``. + ``"not_exists"`` + The source file is only copied if no file of the same path already exists in ``dst_fs``. - If ``condition`` has the value ``"exists"``, the source file is only - copied if a file of the same path already exists in ``dst_fs``. - - If ``condition`` has the value ``"not_exists"``, the source file is only - copied if no file of the same path already exists in ``dst_fs``. Arguments: src_fs (FS or str): Source filesystem (instance or URL). @@ -309,8 +317,10 @@ def copy_dir_if_newer( workers=0, # type: int ): # type: (...) -> None - """Deprecated. Use ``copy_dir_if``.""" - warnings.warn(DeprecationWarning("copy_dir_if_newer is deprecated. Use copy_dir_if instead.")) + """Use ``copy_dir_if`` instead.""" + warnings.warn( + DeprecationWarning("copy_dir_if_newer is deprecated. Use copy_dir_if instead.") + ) copy_dir_if(src_fs, src_path, dst_fs, dst_path, "newer", walker, on_copy, workers) @@ -328,39 +338,36 @@ def copy_dir_if( """Copy a directory from one filesystem to another, depending on a condition. Depending on the value of ``strategy``, certain conditions must be - fulfilled for a file to be copied to ``dst_fs``. - - If ``condition`` has the value ``"always"``, the source file is always - copied. - - If ``condition`` has the value ``"newer"``, the last modification time - of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. - - If ``condition`` has the value ``"older"``, the last modification time - of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. - - If ``condition`` has the value ``"exists"``, the source file is only - copied if a file of the same path already exists in ``dst_fs``. - - If ``condition`` has the value ``"not_exists"``, the source file is only - copied if no file of the same path already exists in ``dst_fs``. + fulfilled for a file to be copied to ``dst_fs``. The following values + are supported: + + ``"always"`` + The source file is always copied. + ``"newer"`` + The last modification time of the source file must be newer than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"older"`` + The last modification time of the source file must be older than that of the destination file. + If either file has no modification time, the copy is performed always. + ``"exists"`` + The source file is only copied if a file of the same path already exists in ``dst_fs``. + ``"not_exists"`` + The source file is only copied if no file of the same path already exists in ``dst_fs``. Arguments: - src_fs (FS or str): Source filesystem (instance or URL). - src_path (str): Path to a directory on the source filesystem. - dst_fs (FS or str): Destination filesystem (instance or URL). - dst_path (str): Path to a directory on the destination filesystem. - condition (str): Name of the condition to check for each file. - walker (~fs.walk.Walker, optional): A walker object that will be - used to scan for files in ``src_fs``. Set this if you only want - to consider a sub-set of the resources in ``src_fs``. - on_copy (callable):A function callback called after a single file copy - is executed. Expected signature is ``(src_fs, src_path, dst_fs, - dst_path)``. - workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for - a single-threaded copy. + src_fs (FS or str): Source filesystem (instance or URL). + src_path (str): Path to a directory on the source filesystem. + dst_fs (FS or str): Destination filesystem (instance or URL). + dst_path (str): Path to a directory on the destination filesystem. + condition (str): Name of the condition to check for each file. + walker (~fs.walk.Walker, optional): A walker object that will be + used to scan for files in ``src_fs``. Set this if you only want + to consider a sub-set of the resources in ``src_fs``. + on_copy (callable):A function callback called after a single file copy + is executed. Expected signature is ``(src_fs, src_path, dst_fs, + dst_path)``. + workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for + a single-threaded copy. """ on_copy = on_copy or (lambda *args: None) @@ -434,4 +441,4 @@ def _copy_is_necessary( return not dst_fs.exists(dst_path) else: - raise ValueError(condition + "is not a valid copy condition.") + raise ValueError("{} is not a valid copy condition.".format(condition)) From 766da0141b3f5d7ca8aafb8c0285c26f461d6cd0 Mon Sep 17 00:00:00 2001 From: atollk Date: Mon, 22 Mar 2021 23:28:29 +0100 Subject: [PATCH 123/309] Fixed codestyle --- fs/copy.py | 52 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 55172afd..815e8b36 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -78,15 +78,19 @@ def copy_fs_if( ``"always"`` The source file is always copied. ``"newer"`` - The last modification time of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be newer than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"older"`` - The last modification time of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be older than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"exists"`` - The source file is only copied if a file of the same path already exists in ``dst_fs``. + The source file is only copied if a file of the same path already + exists in ``dst_fs``. ``"not_exists"`` - The source file is only copied if no file of the same path already exists in ``dst_fs``. + The source file is only copied if no file of the same path already + exists in ``dst_fs``. Arguments: @@ -99,8 +103,8 @@ def copy_fs_if( on_copy (callable):A function callback called after a single file copy is executed. Expected signature is ``(src_fs, src_path, dst_fs, dst_path)``. - workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for - a single-threaded copy. + workers (int): Use ``worker`` threads to copy data, or ``0`` (default) + for a single-threaded copy. """ return copy_dir_if( @@ -169,15 +173,19 @@ def copy_file_if( ``"always"`` The source file is always copied. ``"newer"`` - The last modification time of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be newer than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"older"`` - The last modification time of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be older than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"exists"`` - The source file is only copied if a file of the same path already exists in ``dst_fs``. + The source file is only copied if a file of the same path already + exists in ``dst_fs``. ``"not_exists"`` - The source file is only copied if no file of the same path already exists in ``dst_fs``. + The source file is only copied if no file of the same path already + exists in ``dst_fs``. Arguments: @@ -344,15 +352,19 @@ def copy_dir_if( ``"always"`` The source file is always copied. ``"newer"`` - The last modification time of the source file must be newer than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be newer than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"older"`` - The last modification time of the source file must be older than that of the destination file. - If either file has no modification time, the copy is performed always. + The last modification time of the source file must be older than that + of the destination file. If either file has no modification time, the + copy is performed always. ``"exists"`` - The source file is only copied if a file of the same path already exists in ``dst_fs``. + The source file is only copied if a file of the same path already + exists in ``dst_fs``. ``"not_exists"`` - The source file is only copied if no file of the same path already exists in ``dst_fs``. + The source file is only copied if no file of the same path already + exists in ``dst_fs``. Arguments: src_fs (FS or str): Source filesystem (instance or URL). From 9f008c5a3246ad86e016431fc60a798127ffce60 Mon Sep 17 00:00:00 2001 From: atollk Date: Wed, 24 Mar 2021 19:37:43 +0100 Subject: [PATCH 124/309] Applied changes suggested by code review. --- fs/copy.py | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 815e8b36..22c7a562 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -53,9 +53,13 @@ def copy_fs_if_newer( workers=0, # type: int ): # type: (...) -> None - """Use ``copy_fs_if`` instead.""" + """Copy the contents of one filesystem to another, checking times. + + .. deprecated:: 2.5.0 + Use `~fs.copy_fs_if` with ``condition="newer"`` instead. + """ warnings.warn( - DeprecationWarning("copy_fs_if_newer is deprecated. Use copy_fs_if instead.") + "copy_fs_if_newer is deprecated. Use copy_fs_if instead.", DeprecationWarning ) return copy_fs_if(src_fs, dst_fs, "newer", walker, on_copy, workers) @@ -147,11 +151,14 @@ def copy_file_if_newer( dst_path, # type: Text ): # type: (...) -> bool - """Use ``copy_file_if`` instead.""" + """Copy a file from one filesystem to another, checking times. + + .. deprecated:: 2.5.0 + Use `~fs.copy_file_if` with ``condition="newer"`` instead. + """ warnings.warn( - DeprecationWarning( - "copy_file_if_newer is deprecated. Use copy_file_if instead." - ) + "copy_file_if_newer is deprecated. Use copy_file_if instead.", + DeprecationWarning, ) return copy_file_if(src_fs, src_path, dst_fs, dst_path, "newer") @@ -325,9 +332,13 @@ def copy_dir_if_newer( workers=0, # type: int ): # type: (...) -> None - """Use ``copy_dir_if`` instead.""" + """Copy a directory from one filesystem to another, checking times. + + .. deprecated:: 2.5.0 + Use `~fs.copy_dir_if` with ``condition="newer"`` instead. + """ warnings.warn( - DeprecationWarning("copy_dir_if_newer is deprecated. Use copy_dir_if instead.") + "copy_dir_if_newer is deprecated. Use copy_dir_if instead.", DeprecationWarning ) copy_dir_if(src_fs, src_path, dst_fs, dst_path, "newer", walker, on_copy, workers) @@ -371,15 +382,15 @@ def copy_dir_if( src_path (str): Path to a directory on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a directory on the destination filesystem. - condition (str): Name of the condition to check for each file. - walker (~fs.walk.Walker, optional): A walker object that will be - used to scan for files in ``src_fs``. Set this if you only want - to consider a sub-set of the resources in ``src_fs``. - on_copy (callable):A function callback called after a single file copy - is executed. Expected signature is ``(src_fs, src_path, dst_fs, - dst_path)``. - workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for - a single-threaded copy. + condition (str): Name of the condition to check for each file. + walker (~fs.walk.Walker, optional): A walker object that will be + used to scan for files in ``src_fs``. Set this if you only want + to consider a sub-set of the resources in ``src_fs``. + on_copy (callable):A function callback called after a single file copy + is executed. Expected signature is ``(src_fs, src_path, dst_fs, + dst_path)``. + workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for + a single-threaded copy. """ on_copy = on_copy or (lambda *args: None) From 923dad97a852bdda1c477eb9b3ad3191cf6935ff Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 22:26:33 +0100 Subject: [PATCH 125/309] Test that no new connection is opened with `_manage_ftp` in `FTPFS.upload` --- tests/test_ftpfs.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index b414283d..b0e0c1f0 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -12,7 +12,12 @@ import unittest import uuid -from six import text_type +try: + from unittest import mock +except ImportError: + import mock + +from six import text_type, BytesIO from ftplib import error_perm from ftplib import error_temp @@ -275,6 +280,12 @@ def test_create(self): with open_fs(url, create=True) as ftp_fs: self.assertTrue(ftp_fs.isfile("foo")) + def test_upload_connection(self): + with mock.patch.object(self.fs, "_manage_ftp") as _manage_ftp: + self.fs.upload("foo", BytesIO(b"hello")) + self.assertEqual(self.fs.gettext("foo"), "hello") + _manage_ftp.assert_not_called() + class TestFTPFSNoMLSD(TestFTPFS): def make_fs(self): From 20e260bde377fa68709c122919f0840a69039cbd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 22:28:33 +0100 Subject: [PATCH 126/309] Update `CHANGELOG.md` with contents of #457 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31cb320c..4dfe9058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Make `FTPFile`, `MemoryFile` and `RawWrapper` accept [`array.array`](https://docs.python.org/3/library/array.html) arguments for the `write` and `writelines` methods, as expected by their base class [`io.RawIOBase`](https://docs.python.org/3/library/io.html#io.RawIOBase). - Various documentation issues, including `MemoryFS` docstring not rendering properly. +- Avoid creating a new connection on every call of `FTPFS.upload`. Closes [#455](https://github.com/PyFilesystem/pyfilesystem2/issues/455). ## [2.4.12] - 2021-01-14 From f047b8e9451f291ffc1fd42d3cb9e80826dcdab5 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 01:04:51 +0100 Subject: [PATCH 127/309] Fix `fs.wrap.WrapReadOnly` not blocking the `removetree` method --- fs/wrap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fs/wrap.py b/fs/wrap.py index 8685e9f6..8d939ef7 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -203,6 +203,11 @@ def removedir(self, path): self.check() raise ResourceReadOnly(path) + def removetree(self, path): + # type: (Text) -> None + self.check() + raise ResourceReadOnly(path) + def setinfo(self, path, info): # type: (Text, RawInfo) -> None self.check() From 694631efc4a7de1ffe3fcee330871cc0bd3c0765 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 01:06:56 +0100 Subject: [PATCH 128/309] Fix behaviour of `fs.wrap.WrapCachedDir` methods `isdir` and `isfile` on missing files --- fs/wrap.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/wrap.py b/fs/wrap.py index 8d939ef7..83e14152 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -135,13 +135,17 @@ def getinfo(self, path, namespaces=None): def isdir(self, path): # type: (Text) -> bool - # FIXME(@althonos): this raises an error on non-existing file ! - return self.getinfo(path).is_dir + try: + return self.getinfo(path).is_dir + except ResourceNotFound: + return False def isfile(self, path): # type: (Text) -> bool - # FIXME(@althonos): this raises an error on non-existing file ! - return not self.getinfo(path).is_dir + try: + return not self.getinfo(path).is_dir + except ResourceNotFound: + return False class WrapReadOnly(WrapFS[_F], typing.Generic[_F]): From 99b0ccaba411be9dffa08399f9eaf2687ddbf9d7 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 01:15:36 +0100 Subject: [PATCH 129/309] Improve tests of `fs.wrap` wrappers to make sure caching and read-only work --- fs/wrap.py | 16 ++++ tests/test_wrap.py | 192 ++++++++++++++++++++++++++++++--------------- 2 files changed, 143 insertions(+), 65 deletions(-) diff --git a/fs/wrap.py b/fs/wrap.py index 83e14152..ccb1d8aa 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -92,6 +92,22 @@ class WrapCachedDir(WrapFS[_F], typing.Generic[_F]): """ + # FIXME (@althonos): The caching data structure can very likely be + # improved. With the current implementation, if `scandir` result was + # cached for `namespaces=["details", "access"]`, calling `scandir` + # again only with `names=["details"]` will miss the cache, even though + # we are already storing the totality of the required metadata. + # + # A possible solution would be to replaced the cached with a + # Dict[Text, Dict[Text, Dict[Text, Info]]] + # ^ ^ ^ ^~~ the actual info object + # | | |~~ the path of the directory entry + # | |~~ the namespace of the info + # |~~ the cached directory entry + # + # Furthermore, `listdir` and `filterdir` calls should be cached as well, + # since they can be written as wrappers of `scandir`. + wrap_name = "cached-dir" def __init__(self, wrap_fs): # noqa: D107 diff --git a/tests/test_wrap.py b/tests/test_wrap.py index 4d5cc8c4..1c88bdb7 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -1,97 +1,159 @@ from __future__ import unicode_literals +import operator import unittest +try: + from unittest import mock +except ImportError: + import mock -from fs import errors +import six + +import fs.wrap +import fs.errors from fs import open_fs -from fs import wrap +from fs.info import Info -class TestWrap(unittest.TestCase): - def test_readonly(self): - mem_fs = open_fs("mem://") - fs = wrap.read_only(mem_fs) +class TestWrapReadOnly(unittest.TestCase): - with self.assertRaises(errors.ResourceReadOnly): - fs.open("foo", "w") + def setUp(self): + self.fs = open_fs("mem://") + self.ro = fs.wrap.read_only(self.fs) - with self.assertRaises(errors.ResourceReadOnly): - fs.appendtext("foo", "bar") + def tearDown(self): + self.fs.close() - with self.assertRaises(errors.ResourceReadOnly): - fs.appendbytes("foo", b"bar") + def assertReadOnly(self, func, *args, **kwargs): + self.assertRaises(fs.errors.ResourceReadOnly, func, *args, **kwargs) - with self.assertRaises(errors.ResourceReadOnly): - fs.makedir("foo") + def test_open_w(self): + self.assertReadOnly(self.ro.open, "foo", "w") - with self.assertRaises(errors.ResourceReadOnly): - fs.move("foo", "bar") + def test_appendtext(self): + self.assertReadOnly(self.ro.appendtext, "foo", "bar") - with self.assertRaises(errors.ResourceReadOnly): - fs.openbin("foo", "w") + def test_appendbytes(self): + self.assertReadOnly(self.ro.appendbytes, "foo", b"bar") - with self.assertRaises(errors.ResourceReadOnly): - fs.remove("foo") + def test_makedir(self): + self.assertReadOnly(self.ro.makedir, "foo") - with self.assertRaises(errors.ResourceReadOnly): - fs.removedir("foo") + def test_move(self): + self.assertReadOnly(self.ro.move, "foo", "bar") - with self.assertRaises(errors.ResourceReadOnly): - fs.setinfo("foo", {}) + def test_openbin_w(self): + self.assertReadOnly(self.ro.openbin, "foo", "w") - with self.assertRaises(errors.ResourceReadOnly): - fs.settimes("foo", {}) + def test_remove(self): + self.assertReadOnly(self.ro.remove, "foo") - with self.assertRaises(errors.ResourceReadOnly): - fs.copy("foo", "bar") + def test_removedir(self): + self.assertReadOnly(self.ro.removedir, "foo") - with self.assertRaises(errors.ResourceReadOnly): - fs.create("foo") + def test_removetree(self): + self.assertReadOnly(self.ro.removetree, "foo") - with self.assertRaises(errors.ResourceReadOnly): - fs.writetext("foo", "bar") + def test_setinfo(self): + self.assertReadOnly(self.ro.setinfo, "foo", {}) - with self.assertRaises(errors.ResourceReadOnly): - fs.writebytes("foo", b"bar") + def test_settimes(self): + self.assertReadOnly(self.ro.settimes, "foo", {}) - with self.assertRaises(errors.ResourceReadOnly): - fs.makedirs("foo/bar") + def test_copy(self): + self.assertReadOnly(self.ro.copy, "foo", "bar") - with self.assertRaises(errors.ResourceReadOnly): - fs.touch("foo") + def test_create(self): + self.assertReadOnly(self.ro.create, "foo") - with self.assertRaises(errors.ResourceReadOnly): - fs.upload("foo", None) + def test_writetext(self): + self.assertReadOnly(self.ro.writetext, "foo", "bar") - with self.assertRaises(errors.ResourceReadOnly): - fs.writefile("foo", None) + def test_writebytes(self): + self.assertReadOnly(self.ro.writebytes, "foo", b"bar") - self.assertTrue(mem_fs.isempty("/")) - mem_fs.writebytes("file", b"read me") - with fs.openbin("file") as read_file: - self.assertEqual(read_file.read(), b"read me") + def test_makedirs(self): + self.assertReadOnly(self.ro.makedirs, "foo/bar") - with fs.open("file", "rb") as read_file: - self.assertEqual(read_file.read(), b"read me") + def test_touch(self): + self.assertReadOnly(self.ro.touch, "foo") - def test_cachedir(self): - mem_fs = open_fs("mem://") - mem_fs.makedirs("foo/bar/baz") - mem_fs.touch("egg") + def test_upload(self): + self.assertReadOnly(self.ro.upload, "foo", six.BytesIO()) - fs = wrap.cache_directory(mem_fs) - self.assertEqual(sorted(fs.listdir("/")), ["egg", "foo"]) - self.assertEqual(sorted(fs.listdir("/")), ["egg", "foo"]) - self.assertTrue(fs.isdir("foo")) - self.assertTrue(fs.isdir("foo")) - self.assertTrue(fs.isfile("egg")) - self.assertTrue(fs.isfile("egg")) + def test_writefile(self): + self.assertReadOnly(self.ro.writefile, "foo", six.StringIO()) - self.assertEqual(fs.getinfo("foo"), mem_fs.getinfo("foo")) - self.assertEqual(fs.getinfo("foo"), mem_fs.getinfo("foo")) + def test_openbin_r(self): + self.fs.writebytes("file", b"read me") + with self.ro.openbin("file") as read_file: + self.assertEqual(read_file.read(), b"read me") + + def test_open_r(self): + self.fs.writebytes("file", b"read me") + with self.ro.open("file", "rb") as read_file: + self.assertEqual(read_file.read(), b"read me") - self.assertEqual(fs.getinfo("/"), mem_fs.getinfo("/")) - self.assertEqual(fs.getinfo("/"), mem_fs.getinfo("/")) - with self.assertRaises(errors.ResourceNotFound): - fs.getinfo("/foofoo") +class TestWrapCachedDir(unittest.TestCase): + + def setUp(self): + self.fs = open_fs("mem://") + self.fs.makedirs("foo/bar/baz") + self.fs.touch("egg") + self.cached = fs.wrap.cache_directory(self.fs) + + def tearDown(self): + self.fs.close() + + def assertNotFound(self, func, *args, **kwargs): + self.assertRaises(fs.errors.ResourceNotFound, func, *args, **kwargs) + + def test_scandir(self): + key = operator.attrgetter("name") + expected = [ + Info({"basic": {"name": "egg", "is_dir": False}}), + Info({"basic": {"name": "foo", "is_dir": True}}), + ] + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) + scandir.assert_called() + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) + scandir.assert_not_called() + + def test_isdir(self): + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertTrue(self.cached.isdir("foo")) + self.assertFalse(self.cached.isdir("egg")) # is file + self.assertFalse(self.cached.isdir("spam")) # doesn't exist + scandir.assert_called() + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertTrue(self.cached.isdir("foo")) + self.assertFalse(self.cached.isdir("egg")) + self.assertFalse(self.cached.isdir("spam")) + scandir.assert_not_called() + + def test_isfile(self): + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertTrue(self.cached.isfile("egg")) + self.assertFalse(self.cached.isfile("foo")) # is dir + self.assertFalse(self.cached.isfile("spam")) # doesn't exist + scandir.assert_called() + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertTrue(self.cached.isfile("egg")) + self.assertFalse(self.cached.isfile("foo")) + self.assertFalse(self.cached.isfile("spam")) + scandir.assert_not_called() + + def test_getinfo(self): + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) + self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) + self.assertNotFound(self.cached.getinfo, "spam") + scandir.assert_called() + with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: + self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) + self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) + self.assertNotFound(self.cached.getinfo, "spam") + scandir.assert_not_called() From eb59e5d23adb9ad85ce0d718483d02304fd753c8 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 01:34:37 +0100 Subject: [PATCH 130/309] Add tests checking `WrapReadOnly` works with `fs.copy`, `fs.mirror` and `fs.move` --- fs/wrap.py | 8 +++--- tests/test_wrap.py | 68 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/fs/wrap.py b/fs/wrap.py index ccb1d8aa..3ae4aa9f 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -100,10 +100,10 @@ class WrapCachedDir(WrapFS[_F], typing.Generic[_F]): # # A possible solution would be to replaced the cached with a # Dict[Text, Dict[Text, Dict[Text, Info]]] - # ^ ^ ^ ^~~ the actual info object - # | | |~~ the path of the directory entry - # | |~~ the namespace of the info - # |~~ the cached directory entry + # ^ ^ ^ ^-- the actual info object + # | | \-- the path of the directory entry + # | \-- the namespace of the info + # \-- the cached directory entry # # Furthermore, `listdir` and `filterdir` calls should be cached as well, # since they can be written as wrappers of `scandir`. diff --git a/tests/test_wrap.py b/tests/test_wrap.py index 1c88bdb7..d0fe465c 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -2,6 +2,7 @@ import operator import unittest + try: from unittest import mock except ImportError: @@ -9,6 +10,9 @@ import six +import fs.copy +import fs.mirror +import fs.move import fs.wrap import fs.errors from fs import open_fs @@ -16,7 +20,6 @@ class TestWrapReadOnly(unittest.TestCase): - def setUp(self): self.fs = open_fs("mem://") self.ro = fs.wrap.read_only(self.fs) @@ -95,8 +98,65 @@ def test_open_r(self): self.assertEqual(read_file.read(), b"read me") -class TestWrapCachedDir(unittest.TestCase): +class TestWrapReadOnlySyspath(unittest.TestCase): + # If the wrapped fs has a syspath, there is a chance that somewhere + # in fs.copy or fs.mirror we try to use it to our advantage, but + # we want to make sure these implementations don't circumvent the + # wrapper. + + def setUp(self): + self.fs = open_fs("temp://") + self.ro = fs.wrap.read_only(self.fs) + self.src = open_fs("temp://") + self.src.touch("foo") + self.src.makedir("bar") + + def tearDown(self): + self.fs.close() + self.src.close() + + def assertReadOnly(self, func, *args, **kwargs): + self.assertRaises(fs.errors.ResourceReadOnly, func, *args, **kwargs) + def test_copy_fs(self): + self.assertReadOnly(fs.copy.copy_fs, self.src, self.ro) + + def test_copy_fs_if_newer(self): + self.assertReadOnly(fs.copy.copy_fs_if_newer, self.src, self.ro) + + def test_copy_file(self): + self.assertReadOnly(fs.copy.copy_file, self.src, "foo", self.ro, "foo") + + def test_copy_file_if_newer(self): + self.assertReadOnly(fs.copy.copy_file_if_newer, self.src, "foo", self.ro, "foo") + + def test_copy_structure(self): + self.assertReadOnly(fs.copy.copy_structure, self.src, self.ro) + + def test_mirror(self): + self.assertReadOnly(fs.mirror.mirror, self.src, self.ro) + fs.mirror.mirror(self.src, self.fs) + self.fs.touch("baz") + self.assertReadOnly(fs.mirror.mirror, self.src, self.ro) + + def test_move_fs(self): + self.assertReadOnly(fs.move.move_fs, self.src, self.ro) + self.src.removetree("/") + self.fs.touch("foo") + self.assertReadOnly(fs.move.move_fs, self.ro, self.src) + + def test_move_file(self): + self.assertReadOnly(fs.move.move_file, self.src, "foo", self.ro, "foo") + self.fs.touch("baz") + self.assertReadOnly(fs.move.move_file, self.ro, "baz", self.src, "foo") + + def test_move_dir(self): + self.assertReadOnly(fs.move.move_file, self.src, "bar", self.ro, "bar") + self.fs.makedir("baz") + self.assertReadOnly(fs.move.move_dir, self.ro, "baz", self.src, "baz") + + +class TestWrapCachedDir(unittest.TestCase): def setUp(self): self.fs = open_fs("mem://") self.fs.makedirs("foo/bar/baz") @@ -126,7 +186,7 @@ def test_isdir(self): with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) # is file - self.assertFalse(self.cached.isdir("spam")) # doesn't exist + self.assertFalse(self.cached.isdir("spam")) # doesn't exist scandir.assert_called() with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isdir("foo")) @@ -138,7 +198,7 @@ def test_isfile(self): with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) # is dir - self.assertFalse(self.cached.isfile("spam")) # doesn't exist + self.assertFalse(self.cached.isfile("spam")) # doesn't exist scandir.assert_called() with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isfile("egg")) From 4a614cfc88e31d8c403272508defa6a5e584c0f5 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 01:47:29 +0100 Subject: [PATCH 131/309] Use `assert_has_calls` for compatibility with Python 3.5 in `fs.wrap` --- tests/test_wrap.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_wrap.py b/tests/test_wrap.py index d0fe465c..89a91187 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -177,7 +177,7 @@ def test_scandir(self): ] with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) - scandir.assert_called() + scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) scandir.assert_not_called() @@ -187,7 +187,7 @@ def test_isdir(self): self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) # is file self.assertFalse(self.cached.isdir("spam")) # doesn't exist - scandir.assert_called() + scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) @@ -199,7 +199,7 @@ def test_isfile(self): self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) # is dir self.assertFalse(self.cached.isfile("spam")) # doesn't exist - scandir.assert_called() + scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) @@ -211,7 +211,7 @@ def test_getinfo(self): self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) self.assertNotFound(self.cached.getinfo, "spam") - scandir.assert_called() + scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) From 235b56afec003075f5cef4eee2301cddfda17fab Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 02:06:51 +0100 Subject: [PATCH 132/309] Update `CHANGELOG.md` with `fs.wrap` bugfixes --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dfe9058..0b80ff81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). arguments for the `write` and `writelines` methods, as expected by their base class [`io.RawIOBase`](https://docs.python.org/3/library/io.html#io.RawIOBase). - Various documentation issues, including `MemoryFS` docstring not rendering properly. - Avoid creating a new connection on every call of `FTPFS.upload`. Closes [#455](https://github.com/PyFilesystem/pyfilesystem2/issues/455). +- `WrapReadOnly.removetree` not raising a `ResourceReadOnly` when called. Closes [#468](https://github.com/PyFilesystem/pyfilesystem2/issues/468). +- `WrapCachedDir.isdir` and `WrapCachedDir.isfile` raising a `ResourceNotFound` error on non-existing path ([#470](https://github.com/PyFilesystem/pyfilesystem2/pull/470)). ## [2.4.12] - 2021-01-14 From f13b62ba40fa5149de934105d548332314415f7a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 13:58:55 +0100 Subject: [PATCH 133/309] Implement `MemoryFS.removetree` without having to recurse through the filesystem --- fs/memoryfs.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 30e9c21f..c79ab75e 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -274,6 +274,10 @@ def remove_entry(self, name): # type: (Text) -> None del self._dir[name] + def clear(self): + # type: () -> None + self._dir.clear() + def __contains__(self, name): # type: (object) -> bool return name in self._dir @@ -499,12 +503,29 @@ def remove(self, path): def removedir(self, path): # type: (Text) -> None + # make sure the directory is empty + if not self.isempty(path): + return DirectoryNotEmpty(path) + # make sure we are not removing root _path = self.validatepath(path) - if _path == "/": raise errors.RemoveRootError() + # we can now delegate to removetree since we confirmed that + # * path exists (isempty) + # * path is a folder (isempty) + # * path is not root + self.removetree(_path) + + def removetree(self, path): + # type: (Text) -> None + _path = self.validatepath(path) with self._lock: + + if _path == "/": + self.root.clear() + return + dir_path, file_name = split(_path) parent_dir_entry = self._get_dir_entry(dir_path) @@ -515,9 +536,6 @@ def removedir(self, path): if not dir_dir_entry.is_dir: raise errors.DirectoryExpected(path) - if len(dir_dir_entry): - raise errors.DirectoryNotEmpty(path) - parent_dir_entry.remove_entry(file_name) def setinfo(self, path, info): From 93ea9173cdd58961bc7ac5258dded2ba875a5c6e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 14:33:15 +0100 Subject: [PATCH 134/309] Add dedicated `MemoryFS.scandir` implementation --- fs/memoryfs.py | 62 ++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index c79ab75e..d9a8aa8b 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -5,6 +5,7 @@ import contextlib import io +import itertools import os import time import typing @@ -298,6 +299,21 @@ def remove_open_file(self, memory_file): # type: (_MemoryFile) -> None self._open_files.remove(memory_file) + def to_info(self, namespaces=None): + # type: (Optional[Collection[Text]]) -> Info + namespaces = namespaces or () + info = {"basic": {"name": self.name, "is_dir": self.is_dir}} + if "details" in namespaces: + info["details"] = { + "_write": ["accessed", "modified"], + "type": int(self.resource_type), + "size": self.size, + "accessed": self.accessed_time, + "modified": self.modified_time, + "created": self.created_time, + } + return Info(info) + @six.python_2_unicode_compatible class MemoryFS(FS): @@ -372,33 +388,24 @@ def close(self): def getinfo(self, path, namespaces=None): # type: (Text, Optional[Collection[Text]]) -> Info - namespaces = namespaces or () _path = self.validatepath(path) dir_entry = self._get_dir_entry(_path) if dir_entry is None: raise errors.ResourceNotFound(path) - info = {"basic": {"name": dir_entry.name, "is_dir": dir_entry.is_dir}} - if "details" in namespaces: - info["details"] = { - "_write": ["accessed", "modified"], - "type": int(dir_entry.resource_type), - "size": dir_entry.size, - "accessed": dir_entry.accessed_time, - "modified": dir_entry.modified_time, - "created": dir_entry.created_time, - } - return Info(info) + return dir_entry.to_info(namespaces=namespaces) def listdir(self, path): # type: (Text) -> List[Text] self.check() _path = self.validatepath(path) with self._lock: + # locate and validate the entry corresponding to the given path dir_entry = self._get_dir_entry(_path) if dir_entry is None: raise errors.ResourceNotFound(path) if not dir_entry.is_dir: raise errors.DirectoryExpected(path) + # return the filenames in the order they were created return dir_entry.list() if typing.TYPE_CHECKING: @@ -503,13 +510,13 @@ def remove(self, path): def removedir(self, path): # type: (Text) -> None - # make sure the directory is empty - if not self.isempty(path): - return DirectoryNotEmpty(path) # make sure we are not removing root _path = self.validatepath(path) if _path == "/": raise errors.RemoveRootError() + # make sure the directory is empty + if not self.isempty(path): + raise errors.DirectoryNotEmpty(path) # we can now delegate to removetree since we confirmed that # * path exists (isempty) # * path is a folder (isempty) @@ -538,6 +545,31 @@ def removetree(self, path): parent_dir_entry.remove_entry(file_name) + def scandir( + self, + path, # type: Text + namespaces=None, # type: Optional[Collection[Text]] + page=None, # type: Optional[Tuple[int, int]] + ): + # type: (...) -> Iterator[Info] + self.check() + _path = self.validatepath(path) + with self._lock: + # locate and validate the entry corresponding to the given path + dir_entry = self._get_dir_entry(_path) + if dir_entry is None: + raise errors.ResourceNotFound(path) + if not dir_entry.is_dir: + raise errors.DirectoryExpected(path) + # if paging was requested, slice the filenames + filenames = dir_entry.list() + if page is not None: + start, end = page + filenames = filenames[start:end] + # yield info with the right namespaces + for name in filenames: + yield dir_entry.get_entry(name).to_info(namespaces=namespaces) + def setinfo(self, path, info): # type: (Text, RawInfo) -> None _path = self.validatepath(path) From 3f4fb6c05e3bce9126caff5d43dde773cc7c7bff Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 14:47:34 +0100 Subject: [PATCH 135/309] Implement `MemoryFS.move` and `MemoryFS.movedir` without copy --- fs/memoryfs.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index d9a8aa8b..203350f4 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -444,6 +444,46 @@ def makedir( parent_dir.set_entry(dir_name, new_dir) return self.opendir(path) + def move(self, src_path, dst_path, overwrite=False): + src_dir, src_name = split(self.validatepath(src_path)) + dst_dir, dst_name = split(self.validatepath(dst_path)) + + with self._lock: + src_dir_entry = self._get_dir_entry(src_dir) + if src_dir_entry is None or src_name not in src_dir_entry: + raise errors.ResourceNotFound(src_path) + src_entry = src_dir_entry.get_entry(src_name) + if src_entry.is_dir: + raise errors.FileExpected(src_path) + + dst_dir_entry = self._get_dir_entry(dst_dir) + if dst_dir_entry is None: + raise errors.ResourceNotFound(dst_path) + elif not overwrite and dst_name in dst_dir_entry: + raise errors.DestinationExists(dst_path) + + dst_dir_entry.set_entry(dst_name, src_entry) + src_dir_entry.remove_entry(src_name) + + def movedir(self, src_path, dst_path, create=False): + src_dir, src_name = split(self.validatepath(src_path)) + dst_dir, dst_name = split(self.validatepath(dst_path)) + + with self._lock: + src_dir_entry = self._get_dir_entry(src_dir) + if src_dir_entry is None or src_name not in src_dir_entry: + raise errors.ResourceNotFound(src_path) + src_entry = src_dir_entry.get_entry(src_name) + if not src_entry.is_dir: + raise errors.DirectoryExpected(src_path) + + dst_dir_entry = self._get_dir_entry(dst_dir) + if dst_dir_entry is None or (not create and dst_name not in dst_dir_entry): + raise errors.ResourceNotFound(dst_path) + + dst_dir_entry.set_entry(dst_name, src_entry) + src_dir_entry.remove_entry(src_name) + def openbin(self, path, mode="r", buffering=-1, **options): # type: (Text, Text, int, **Any) -> BinaryIO _mode = Mode(mode) From 97d5b77fbba00de700fcfe127cc8d66a0e3f1c7b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 14:59:58 +0100 Subject: [PATCH 136/309] Update `CHANGELOG.md` with `MemoryFS` improvements --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b80ff81..c99ffcc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,11 +20,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/). Closes [#445](https://github.com/PyFilesystem/pyfilesystem2/pull/445). - Migrate continuous integration from Travis-CI to GitHub Actions and introduce several linters again in the build steps ([#448](https://github.com/PyFilesystem/pyfilesystem2/pull/448)). - Closes [#446](https://github.com/PyFilesystem/pyfilesystem2/pull/446). + Closes [#446](https://github.com/PyFilesystem/pyfilesystem2/issues/446). - Stop requiring `pytest` to run tests, allowing any test runner supporting `unittest`-style test suites. - `FSTestCases` now builds the large data required for `upload` and `download` tests only once in order to reduce the total testing time. +- `MemoryFS.move` and `MemoryFS.movedir` will now avoid copying data. + Closes [#452](https://github.com/PyFilesystem/pyfilesystem2/issues/452). ### Fixed From eb620266c7430ab98688131a531bf56b16a8030e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 15:20:13 +0100 Subject: [PATCH 137/309] Fix typing and unused imports in `fs.memoryfs` --- fs/memoryfs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 203350f4..7965a135 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -5,7 +5,6 @@ import contextlib import io -import itertools import os import time import typing @@ -39,6 +38,7 @@ SupportsInt, Union, Text, + Tuple, ) from .base import _OpendirFactory from .info import RawInfo @@ -608,7 +608,8 @@ def scandir( filenames = filenames[start:end] # yield info with the right namespaces for name in filenames: - yield dir_entry.get_entry(name).to_info(namespaces=namespaces) + entry = typing.cast(_DirEntry, dir_entry.get_entry(name)) + yield entry.to_info(namespaces=namespaces) def setinfo(self, path, info): # type: (Text, RawInfo) -> None From f26b7d14ba00bb443562c986a9c3d7ba453c4c5c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 15:44:43 +0100 Subject: [PATCH 138/309] Add test to cover missed branches in `fs.memoryfs` --- fs/test.py | 9 +++++++++ tests/test_memoryfs.py | 17 +++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/fs/test.py b/fs/test.py index d8b5b812..90c9131d 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1115,6 +1115,15 @@ def test_removetree(self): self.fs.removetree("foo") self.assert_not_exists("foo") + # Errors on files + self.fs.create("bar") + with self.assertRaises(errors.DirectoryExpected): + self.fs.removetree("bar") + + # Errors on non-existing path + with self.assertRaises(errors.ResourceNotFound): + self.fs.removetree("foofoo") + def test_setinfo(self): self.fs.create("birthday.txt") now = math.floor(time.time()) diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 0b26a576..31909f0f 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -66,3 +66,20 @@ def test_close_mem_free(self): "Memory usage increased after closing the file system; diff is %0.2f KiB." % (diff_close.size_diff / 1024.0), ) + + +class TestMemoryFile(unittest.TestCase): + + def setUp(self): + self.fs = memoryfs.MemoryFS() + + def tearDown(self): + self.fs.close() + + def test_readline_writing(self): + with self.fs.openbin("test.txt", "w") as f: + self.assertRaises(IOError, f.readline) + + def test_readinto_writing(self): + with self.fs.openbin("test.txt", "w") as f: + self.assertRaises(IOError, f.readinto, bytearray(10)) From 9ec66bf9ab44dad92fcbd5c11d59da887fdf99dd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 16:38:21 +0100 Subject: [PATCH 139/309] Make GitHub Actions only verify coverage of all jobs combined --- .github/workflows/test.yml | 28 ++++++++++++++++++++++------ setup.cfg | 4 ++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52516c34..3b8a4932 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,20 +32,36 @@ jobs: run: python -m pip install tox tox-gh-actions - name: Test with tox run: python -m tox - - name: Collect coverage results - uses: AndreMiras/coveralls-python-action@develop + - name: Store partial coverage reports + uses: actions/upload-artifact@v2 with: - parallel: true - flag-name: test (${{ matrix.python-version}}) + name: coverage + path: .coverage.* coveralls: needs: test runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Setup Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install coverage package + run: python -m pip install -U coverage + - name: Download partial coverage reports + uses: actions/download-artifact@v2 + with: + name: coverage + - name: Combine coverage + run: python -m coverage combine + - name: Report coverage + run: python -m coverage report + - name: Export coverage to XML + run: python -m coverage xml - name: Upload coverage statistics to Coveralls uses: AndreMiras/coveralls-python-action@develop - with: - parallel-finished: true lint: runs-on: ubuntu-latest diff --git a/setup.cfg b/setup.cfg index b90ebe47..11cbb16f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -111,6 +111,7 @@ branch = true omit = fs/test.py source = fs relative_files = true +parallel = true [coverage:report] show_missing = true @@ -135,10 +136,9 @@ requires = setuptools >=38.3.0 [testenv] -commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests +commands = python -m coverage run --rcfile {toxinidir}/setup.cfg -m pytest {posargs} {toxinidir}/tests deps = -rtests/requirements.txt - pytest-cov~=2.11 coverage~=5.0 py{36,37,38,39}: pytest~=6.2 py{27,34}: pytest~=4.6 From df4fb237531f110e4620df6ea7799c1598c7b0f1 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 16:44:17 +0100 Subject: [PATCH 140/309] Fix `pytest` not being installed in `py35` tox environment --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 11cbb16f..5189720c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -140,9 +140,9 @@ commands = python -m coverage run --rcfile {toxinidir}/setup.cfg -m pytest {posa deps = -rtests/requirements.txt coverage~=5.0 - py{36,37,38,39}: pytest~=6.2 + py{35,36,37,38,39}: pytest~=6.0 py{27,34}: pytest~=4.6 - py{36,37,38,39}: pytest-randomly~=3.5 + py{35,36,37,38,39}: pytest-randomly~=3.5 py{27,34}: pytest-randomly~=1.2 scandir: .[scandir] !scandir: . From 4efe083ebb1a43aafd0e171b3b20d6b546c86756 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 16:47:20 +0100 Subject: [PATCH 141/309] Fix `pytest` not being installed in several PyPy tox environment --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5189720c..a5c46904 100644 --- a/setup.cfg +++ b/setup.cfg @@ -140,10 +140,10 @@ commands = python -m coverage run --rcfile {toxinidir}/setup.cfg -m pytest {posa deps = -rtests/requirements.txt coverage~=5.0 - py{35,36,37,38,39}: pytest~=6.0 - py{27,34}: pytest~=4.6 - py{35,36,37,38,39}: pytest-randomly~=3.5 - py{27,34}: pytest-randomly~=1.2 + py{35,36,37,38,39,py36,py37}: pytest~=6.0 + py{27,34,py27}: pytest~=4.6 + py{35,36,37,38,39,py36,py37}: pytest-randomly~=3.5 + py{27,34,py27}: pytest-randomly~=1.2 scandir: .[scandir] !scandir: . From 3df9836cd3aa5f3d99633df93e30040a9c937696 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 19:24:03 +0100 Subject: [PATCH 142/309] Fix regex for Linux FTP not recognizing sticky/SUID/SGID permissions --- fs/_ftp_parse.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 9e15a265..a9088ab4 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -19,7 +19,8 @@ RE_LINUX = re.compile( r""" ^ - ([ldrwx-]{10}) + ([-dlpscbD]) + ([r-][w-][xsS-][r-][w-][xsS-][r-][w-][xtT-][\.\+]?) \s+? (\d+) \s+? @@ -110,14 +111,14 @@ def _decode_linux_time(mtime): def decode_linux(line, match): - perms, links, uid, gid, size, mtime, name = match.groups() - is_link = perms.startswith("l") - is_dir = perms.startswith("d") or is_link + ty, perms, links, uid, gid, size, mtime, name = match.groups() + is_link = ty == "l" + is_dir = ty == "d" or is_link if is_link: name, _, _link_name = name.partition("->") name = name.strip() _link_name = _link_name.strip() - permissions = Permissions.parse(perms[1:]) + permissions = Permissions.parse(perms) mtime_epoch = _decode_linux_time(mtime) From bbaa04102f58bd69c9afc6b14c1e51f2f3cbb805 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 19:24:35 +0100 Subject: [PATCH 143/309] Add regression test for #451 to `tests.test_ftp_parse` --- tests/test_ftp_parse.py | 146 +++++++++++++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 26 deletions(-) diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index b9a69cf1..fe8e0e49 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import textwrap import time import unittest @@ -33,7 +34,7 @@ def test_parse_time(self, mock_localtime): self.assertEqual(ftp_parse._parse_time("notadate", formats=["%b %d %Y"]), None) def test_parse(self): - self.assertEqual(ftp_parse.parse([""]), []) + self.assertListEqual(ftp_parse.parse([""]), []) def test_parse_line(self): self.assertIs(ftp_parse.parse_line("not a dir"), None) @@ -41,15 +42,17 @@ def test_parse_line(self): @mock.patch("time.localtime") def test_decode_linux(self, mock_localtime): mock_localtime.return_value = time2017 - directory = """\ -lrwxrwxrwx 1 0 0 19 Jan 18 2006 debian -> ./pub/mirror/debian -drwxr-xr-x 10 0 0 4096 Aug 03 09:21 debian-archive -lrwxrwxrwx 1 0 0 27 Nov 30 2015 debian-backports -> pub/mirror/debian-backports -drwxr-xr-x 12 0 0 4096 Sep 29 13:13 pub --rw-r--r-- 1 0 0 26 Mar 04 2010 robots.txt -drwxr-xr-x 8 foo bar 4096 Oct 4 09:05 test -drwxr-xr-x 2 foo-user foo-group 0 Jan 5 11:59 240485 -""" + directory = textwrap.dedent( + """ + lrwxrwxrwx 1 0 0 19 Jan 18 2006 debian -> ./pub/mirror/debian + drwxr-xr-x 10 0 0 4096 Aug 03 09:21 debian-archive + lrwxrwxrwx 1 0 0 27 Nov 30 2015 debian-backports -> pub/mirror/debian-backports + drwxr-xr-x 12 0 0 4096 Sep 29 13:13 pub + -rw-r--r-- 1 0 0 26 Mar 04 2010 robots.txt + drwxr-xr-x 8 foo bar 4096 Oct 4 09:05 test + drwxr-xr-x 2 foo-user foo-group 0 Jan 5 11:59 240485 + """ + ) expected = [ { @@ -158,25 +161,27 @@ def test_decode_linux(self, mock_localtime): }, ] - parsed = ftp_parse.parse(directory.splitlines()) - self.assertEqual(parsed, expected) + parsed = ftp_parse.parse(directory.strip().splitlines()) + self.assertListEqual(parsed, expected) @mock.patch("time.localtime") def test_decode_windowsnt(self, mock_localtime): mock_localtime.return_value = time2017 - directory = """\ -unparsable line -11-02-17 02:00AM docs -11-02-17 02:12PM images -11-02-17 02:12PM AM to PM -11-02-17 03:33PM 9276 logo.gif -05-11-20 22:11 src -11-02-17 01:23 1 12 -11-02-17 4:54 0 icon.bmp -11-02-17 4:54AM 0 icon.gif -11-02-17 4:54PM 0 icon.png -11-02-17 16:54 0 icon.jpg -""" + directory = textwrap.dedent( + """ + unparsable line + 11-02-17 02:00AM docs + 11-02-17 02:12PM images + 11-02-17 02:12PM AM to PM + 11-02-17 03:33PM 9276 logo.gif + 05-11-20 22:11 src + 11-02-17 01:23 1 12 + 11-02-17 4:54 0 icon.bmp + 11-02-17 4:54AM 0 icon.gif + 11-02-17 4:54PM 0 icon.png + 11-02-17 16:54 0 icon.jpg + """ + ) expected = [ { "basic": {"is_dir": True, "name": "docs"}, @@ -230,5 +235,94 @@ def test_decode_windowsnt(self, mock_localtime): }, ] - parsed = ftp_parse.parse(directory.splitlines()) + parsed = ftp_parse.parse(directory.strip().splitlines()) self.assertEqual(parsed, expected) + + @mock.patch("time.localtime") + def test_decode_linux_suid(self, mock_localtime): + # reported in #451 + mock_localtime.return_value = time2017 + directory = textwrap.dedent( + """ + drwxr-sr-x 66 ftp ftp 8192 Mar 16 17:54 pub + -rw-r--r-- 1 ftp ftp 25 Mar 18 19:34 robots.txt + """ + ) + expected = [ + { + "access": { + "group": "ftp", + "permissions": [ + "g_r", + "g_s", + "o_r", + "o_x", + "u_r", + "u_w", + "u_x", + ], + "user": "ftp", + }, + "basic": {"is_dir": True, "name": "pub"}, + "details": {"modified": 1489686840.0, "size": 8192, "type": 1}, + "ftp": { + "ls": "drwxr-sr-x 66 ftp ftp 8192 Mar 16 17:54 pub" + }, + }, + { + "access": { + "group": "ftp", + "permissions": [ + "g_r", + "o_r", + "u_r", + "u_w", + ], + "user": "ftp", + }, + "basic": {"is_dir": False, "name": "robots.txt"}, + "details": {"modified": 1489865640.0, "size": 25, "type": 2}, + "ftp": { + "ls": "-rw-r--r-- 1 ftp ftp 25 Mar 18 19:34 robots.txt" + }, + } + ] + + parsed = ftp_parse.parse(directory.strip().splitlines()) + self.assertListEqual(parsed, expected) + + @mock.patch("time.localtime") + def test_decode_linux_sticky(self, mock_localtime): + # reported in #451 + mock_localtime.return_value = time2017 + directory = textwrap.dedent( + """ + drwxr-xr-t 66 ftp ftp 8192 Mar 16 17:54 pub + """ + ) + expected = [ + { + "access": { + "group": "ftp", + "permissions": [ + "g_r", + "g_x", + "o_r", + "o_t", + "u_r", + "u_w", + "u_x", + ], + "user": "ftp", + }, + "basic": {"is_dir": True, "name": "pub"}, + "details": {"modified": 1489686840.0, "size": 8192, "type": 1}, + "ftp": { + "ls": "drwxr-xr-t 66 ftp ftp 8192 Mar 16 17:54 pub" + }, + }, + ] + + self.maxDiff = None + parsed = ftp_parse.parse(directory.strip().splitlines()) + self.assertListEqual(parsed, expected) From 7f764542a7bc846fe5feb37ca1bcb852cdc0ef76 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 19:39:20 +0100 Subject: [PATCH 144/309] Update `CHANGELOG.md` with `fs._ftp_parse` bugfix --- CHANGELOG.md | 4 +++- tests/test_ftp_parse.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c99ffcc3..aacded93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). test suites. - `FSTestCases` now builds the large data required for `upload` and `download` tests only once in order to reduce the total testing time. -- `MemoryFS.move` and `MemoryFS.movedir` will now avoid copying data. +- `MemoryFS.move` and `MemoryFS.movedir` will now avoid copying data. Closes [#452](https://github.com/PyFilesystem/pyfilesystem2/issues/452). ### Fixed @@ -36,6 +36,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Avoid creating a new connection on every call of `FTPFS.upload`. Closes [#455](https://github.com/PyFilesystem/pyfilesystem2/issues/455). - `WrapReadOnly.removetree` not raising a `ResourceReadOnly` when called. Closes [#468](https://github.com/PyFilesystem/pyfilesystem2/issues/468). - `WrapCachedDir.isdir` and `WrapCachedDir.isfile` raising a `ResourceNotFound` error on non-existing path ([#470](https://github.com/PyFilesystem/pyfilesystem2/pull/470)). +- `FTPFS` not listing certain entries with sticky/SUID/SGID permissions set by Linux server ([#473](https://github.com/PyFilesystem/pyfilesystem2/pull/473)). + Closes [#451](https://github.com/PyFilesystem/pyfilesystem2/issues/451). ## [2.4.12] - 2021-01-14 diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index fe8e0e49..d027082d 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -285,7 +285,7 @@ def test_decode_linux_suid(self, mock_localtime): "ftp": { "ls": "-rw-r--r-- 1 ftp ftp 25 Mar 18 19:34 robots.txt" }, - } + }, ] parsed = ftp_parse.parse(directory.strip().splitlines()) From f412cb4fcb48576bdf6d9ef3e43516baa7259f29 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 03:10:16 +0100 Subject: [PATCH 145/309] Add `exc` keyword and attribute to `fs.errors.PathError` --- fs/errors.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/errors.py b/fs/errors.py index 25625e28..2448c7a6 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -139,13 +139,14 @@ class PathError(FSError): default_message = "path '{path}' is invalid" - def __init__(self, path, msg=None): # noqa: D107 - # type: (Text, Optional[Text]) -> None + def __init__(self, path, msg=None, exc=None): # noqa: D107 + # type: (Text, Optional[Text], Optional[Exception]) -> None self.path = path + self.exc = exc super(PathError, self).__init__(msg=msg) def __reduce__(self): - return type(self), (self.path, self._msg) + return type(self), (self.path, self._msg, self.exc) class NoSysPath(PathError): From eebb4a2318263df79081380b1ae55cffda27315f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 03:21:59 +0100 Subject: [PATCH 146/309] Add regression test for #453 and update `CHANGELOG.md` --- CHANGELOG.md | 2 ++ tests/test_error_tools.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aacded93..62cc085c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added FTP over TLS (FTPS) support to FTPFS. Closes [#437](https://github.com/PyFilesystem/pyfilesystem2/issues/437), [#449](https://github.com/PyFilesystem/pyfilesystem2/pull/449). +- `PathError` now supports wrapping an exception using the `exc` argument. + Closes [#453](https://github.com/PyFilesystem/pyfilesystem2/issues/453). ### Changed diff --git a/tests/test_error_tools.py b/tests/test_error_tools.py index b9ac25c9..4f6aa324 100644 --- a/tests/test_error_tools.py +++ b/tests/test_error_tools.py @@ -3,13 +3,23 @@ import errno import unittest +import fs.errors from fs.error_tools import convert_os_errors -from fs import errors as fserrors class TestErrorTools(unittest.TestCase): - def assert_convert_os_errors(self): + def test_convert_enoent(self): + exception = OSError(errno.ENOENT, "resource not found") + with self.assertRaises(fs.errors.ResourceNotFound) as ctx: + with convert_os_errors("stat", "/tmp/test"): + raise exception + self.assertEqual(ctx.exception.exc, exception) + self.assertEqual(ctx.exception.path, "/tmp/test") - with self.assertRaises(fserrors.ResourceNotFound): - with convert_os_errors("foo", "test"): - raise OSError(errno.ENOENT) + def test_convert_enametoolong(self): + exception = OSError(errno.ENAMETOOLONG, "File name too long: test") + with self.assertRaises(fs.errors.PathError) as ctx: + with convert_os_errors("stat", "/tmp/test"): + raise exception + self.assertEqual(ctx.exception.exc, exception) + self.assertEqual(ctx.exception.path, "/tmp/test") From 67a8ce9afe845996f6479070d56f0542ee3cc03d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 04:05:17 +0100 Subject: [PATCH 147/309] Patch `OSFS.scandir` so it properly closes the `scandir` iterator --- fs/osfs.py | 69 +++++++++++++++++++++++++--------------------- tests/test_osfs.py | 9 ++++++ 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 3b35541c..cafdff05 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -483,39 +483,44 @@ def _scandir(self, path, namespaces=None): else: sys_path = self._to_sys_path(_path) # type: ignore with convert_os_errors("scandir", path, directory=True): - for dir_entry in scandir(sys_path): - info = { - "basic": { - "name": fsdecode(dir_entry.name), - "is_dir": dir_entry.is_dir(), - } - } - if "details" in namespaces: - stat_result = dir_entry.stat() - info["details"] = self._make_details_from_stat(stat_result) - if "stat" in namespaces: - stat_result = dir_entry.stat() - info["stat"] = { - k: getattr(stat_result, k) - for k in dir(stat_result) - if k.startswith("st_") - } - if "lstat" in namespaces: - lstat_result = dir_entry.stat(follow_symlinks=False) - info["lstat"] = { - k: getattr(lstat_result, k) - for k in dir(lstat_result) - if k.startswith("st_") + scandir_iter = scandir(sys_path) + try: + for dir_entry in scandir_iter: + info = { + "basic": { + "name": fsdecode(dir_entry.name), + "is_dir": dir_entry.is_dir(), + } } - if "link" in namespaces: - info["link"] = self._make_link_info( - os.path.join(sys_path, dir_entry.name) - ) - if "access" in namespaces: - stat_result = dir_entry.stat() - info["access"] = self._make_access_from_stat(stat_result) - - yield Info(info) + if "details" in namespaces: + stat_result = dir_entry.stat() + info["details"] = self._make_details_from_stat(stat_result) + if "stat" in namespaces: + stat_result = dir_entry.stat() + info["stat"] = { + k: getattr(stat_result, k) + for k in dir(stat_result) + if k.startswith("st_") + } + if "lstat" in namespaces: + lstat_result = dir_entry.stat(follow_symlinks=False) + info["lstat"] = { + k: getattr(lstat_result, k) + for k in dir(lstat_result) + if k.startswith("st_") + } + if "link" in namespaces: + info["link"] = self._make_link_info( + os.path.join(sys_path, dir_entry.name) + ) + if "access" in namespaces: + stat_result = dir_entry.stat() + info["access"] = self._make_access_from_stat(stat_result) + + yield Info(info) + finally: + if sys.version_info >= (3, 6): + scandir_iter.close() else: diff --git a/tests/test_osfs.py b/tests/test_osfs.py index e43635f4..88879ec9 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -8,6 +8,7 @@ import tempfile import sys import unittest +import warnings from fs import osfs, open_fs from fs.path import relpath, dirname @@ -25,6 +26,14 @@ class TestOSFS(FSTestCases, unittest.TestCase): """Test OSFS implementation.""" + @classmethod + def setUpClass(cls): + warnings.simplefilter("error") + + @classmethod + def tearDownClass(cls): + warnings.simplefilter(warnings.defaultaction) + def make_fs(self): temp_dir = tempfile.mkdtemp("fstestosfs") return osfs.OSFS(temp_dir) From 7ed4fb5be71af080a99e3971b6ca05a06777d4ac Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 04:08:00 +0100 Subject: [PATCH 148/309] Fix `OSFS.scandir` calling `os.stat` up to 3 times when several namespaces are requested --- fs/osfs.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index cafdff05..e840ffc4 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -475,6 +475,7 @@ def _scandir(self, path, namespaces=None): # type: (Text, Optional[Collection[Text]]) -> Iterator[Info] self.check() namespaces = namespaces or () + requires_stat = not {"details", "stat", "access"}.isdisjoint(namespaces) _path = self.validatepath(path) if _WINDOWS_PLATFORM: sys_path = os.path.join( @@ -492,16 +493,18 @@ def _scandir(self, path, namespaces=None): "is_dir": dir_entry.is_dir(), } } - if "details" in namespaces: + if requires_stat: stat_result = dir_entry.stat() - info["details"] = self._make_details_from_stat(stat_result) - if "stat" in namespaces: - stat_result = dir_entry.stat() - info["stat"] = { - k: getattr(stat_result, k) - for k in dir(stat_result) - if k.startswith("st_") - } + if "details" in namespaces: + info["details"] = self._make_details_from_stat(stat_result) + if "stat" in namespaces: + info["stat"] = { + k: getattr(stat_result, k) + for k in dir(stat_result) + if k.startswith("st_") + } + if "access" in namespaces: + info["access"] = self._make_access_from_stat(stat_result) if "lstat" in namespaces: lstat_result = dir_entry.stat(follow_symlinks=False) info["lstat"] = { @@ -513,10 +516,7 @@ def _scandir(self, path, namespaces=None): info["link"] = self._make_link_info( os.path.join(sys_path, dir_entry.name) ) - if "access" in namespaces: - stat_result = dir_entry.stat() - info["access"] = self._make_access_from_stat(stat_result) - + yield Info(info) finally: if sys.version_info >= (3, 6): From ea75f073641df1694ea8adddb00139f1c3ec829c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 04:11:00 +0100 Subject: [PATCH 149/309] Fix `ResourceWarning` caused by unclosed files in `FSTestCase.test_files` --- fs/test.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fs/test.py b/fs/test.py index 90c9131d..4d4e4518 100644 --- a/fs/test.py +++ b/fs/test.py @@ -16,6 +16,7 @@ import os import time import unittest +import warnings import fs.copy import fs.move @@ -884,8 +885,9 @@ def test_open_files(self): self.assertFalse(f.closed) self.assertTrue(f.closed) - iter_lines = iter(self.fs.open("text")) - self.assertEqual(next(iter_lines), "Hello\n") + with self.fs.open("text") as f: + iter_lines = iter(f) + self.assertEqual(next(iter_lines), "Hello\n") with self.fs.open("unicode", "w") as f: self.assertEqual(12, f.write("Héllo\nWörld\n")) @@ -1594,8 +1596,10 @@ def test_files(self): self.assert_bytes("foo2", b"help") # Test __del__ doesn't throw traceback - f = self.fs.open("foo2", "r") - del f + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + f = self.fs.open("foo2", "r") + del f with self.assertRaises(IOError): with self.fs.open("foo2", "r") as f: From 8aff8bef3fcbebd41b90397aa2171c5499b5a29d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 04:24:45 +0100 Subject: [PATCH 150/309] Reformat code in `fs.osfs` and add entry to `CHANGELOG.md` --- CHANGELOG.md | 2 ++ fs/osfs.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62cc085c..f7b98bc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `WrapCachedDir.isdir` and `WrapCachedDir.isfile` raising a `ResourceNotFound` error on non-existing path ([#470](https://github.com/PyFilesystem/pyfilesystem2/pull/470)). - `FTPFS` not listing certain entries with sticky/SUID/SGID permissions set by Linux server ([#473](https://github.com/PyFilesystem/pyfilesystem2/pull/473)). Closes [#451](https://github.com/PyFilesystem/pyfilesystem2/issues/451). +- `scandir` iterator not being closed explicitly in `OSFS.scandir`, occasionally causing a `ResourceWarning` + to be thrown. Closes [#311](https://github.com/PyFilesystem/pyfilesystem2/issues/311). ## [2.4.12] - 2021-01-14 diff --git a/fs/osfs.py b/fs/osfs.py index e840ffc4..5beb16bf 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -496,7 +496,9 @@ def _scandir(self, path, namespaces=None): if requires_stat: stat_result = dir_entry.stat() if "details" in namespaces: - info["details"] = self._make_details_from_stat(stat_result) + info["details"] = self._make_details_from_stat( + stat_result + ) if "stat" in namespaces: info["stat"] = { k: getattr(stat_result, k) @@ -504,7 +506,9 @@ def _scandir(self, path, namespaces=None): if k.startswith("st_") } if "access" in namespaces: - info["access"] = self._make_access_from_stat(stat_result) + info["access"] = self._make_access_from_stat( + stat_result + ) if "lstat" in namespaces: lstat_result = dir_entry.stat(follow_symlinks=False) info["lstat"] = { From 47b6239560696aa040134284b59e22fdbc953226 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 15:36:42 +0100 Subject: [PATCH 151/309] Add disclaimer about the `writeable` parameter of `fs.open_fs` --- fs/opener/registry.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 4c1c2d3e..f923b4ea 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -198,7 +198,8 @@ def open_fs( """Open a filesystem from a FS URL (ignoring the path component). Arguments: - fs_url (str): A filesystem URL. + fs_url (str): A filesystem URL. If a filesystem instance is + given instead, it will be returned transparently. writeable (bool, optional): `True` if the filesystem must be writeable. create (bool, optional): `True` if the filesystem should be @@ -211,6 +212,14 @@ def open_fs( Returns: ~fs.base.FS: A filesystem instance. + Caution: + The ``writeable`` parameter only controls whether the + filesystem *needs* to be writable, which is relevant for + some archive filesystems. Passing ``writeable=False`` will + **not** make the return filesystem read-only. For this, + consider using `fs.wrap.WrapReadOnly` to wrap the returned + instance. + """ from ..base import FS From 8e7a2e9192dadbcfd02acd10bcd7e05357af32e8 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:03:07 +0100 Subject: [PATCH 152/309] Add a loader for doctests to the unittest suite --- tests/test_doctest.py | 177 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 tests/test_doctest.py diff --git a/tests/test_doctest.py b/tests/test_doctest.py new file mode 100644 index 00000000..c96cb84b --- /dev/null +++ b/tests/test_doctest.py @@ -0,0 +1,177 @@ +# coding: utf-8 +"""Test doctest contained tests in every file of the module. +""" +import doctest +import importlib +import os +import pkgutil +import sys +import types +import warnings +import tempfile + +try: + from unittest import mock +except ImportError: + import mock + +import six + +import fs +import fs.opener.parse +from fs.memoryfs import MemoryFS +from fs.subfs import ClosingSubFS +from fs.tempfs import TempFS + +# --- Mocks ------------------------------------------------------------------ + + +def _home_fs(): + """Create a mock filesystem that matches the XDG user-dirs spec.""" + home_fs = MemoryFS() + home_fs.makedir("Desktop") + home_fs.makedir("Documents") + home_fs.makedir("Downloads") + home_fs.makedir("Music") + home_fs.makedir("Pictures") + home_fs.makedir("Public") + home_fs.makedir("Templates") + home_fs.makedir("Videos") + return home_fs + + +def _open_fs(path): + """A mock `open_fs` that avoids side effects when running doctests.""" + if "://" not in path: + path = "osfs://{}".format(path) + parse_result = fs.opener.parse(path) + if parse_result.protocol == "osfs" and parse_result.resource == "~": + home_fs = _home_fs() + if parse_result.path is not None: + home_fs = home_fs.opendir(parse_result.path, factory=ClosingSubFS) + return home_fs + elif parse_result.protocol in {"ftp", "ftps", "mem"}: + return MemoryFS() + else: + raise RuntimeError("not allowed in doctests: {}".format(path)) + + +def _my_fs(module): + """Create a mock filesystem to be used in examples.""" + my_fs = MemoryFS() + + if module == "fs.base": + my_fs.makedir("Desktop") + my_fs.makedir("Videos") + my_fs.touch("Videos/starwars.mov") + my_fs.touch("file.txt") + + elif module == "fs.info": + my_fs.touch("foo.tar.gz") + my_fs.settext("foo.py", "print('Hello, world!')") + my_fs.makedir("bar") + + elif module in {"fs.walk", "fs.glob"}: + my_fs.makedir("dir1") + my_fs.makedir("dir2") + my_fs.settext("foo.py", "print('Hello, world!')") + my_fs.touch("foo.pyc") + my_fs.settext("bar.py", "print('ok')\n\n# this is a comment\n") + my_fs.touch("bar.pyc") + + # # used in `fs.glob` + # home_fs.touch("foo.pyc") + # home_fs.touch("bar.pyc") + return my_fs + + +def _open(filename, mode="r"): + """A mock `open` that actually opens a temporary file.""" + return tempfile.NamedTemporaryFile(mode="r+" if mode == "r" else mode) + + +# --- Loader protocol -------------------------------------------------------- + + +def _load_tests_from_module(tests, module, globs, setUp=None, tearDown=None): + """Load tests from module, iterating through submodules.""" + for attr in (getattr(module, x) for x in dir(module) if not x.startswith("_")): + if isinstance(attr, types.ModuleType): + suite = doctest.DocTestSuite( + attr, + globs, + setUp=setUp, + tearDown=tearDown, + optionflags=+doctest.ELLIPSIS, + ) + tests.addTests(suite) + return tests + + +def load_tests(loader, tests, ignore): + """`load_test` function used by unittest to find the doctests.""" + + # NB (@althonos): we only test docstrings on Python 3 because it's + # extremely hard to maintain compatibility for both versions without + # extensively hacking `doctest` and `unittest`. + if six.PY2: + return tests + + def setUp(self): + warnings.simplefilter("ignore") + self._open_fs_mock = mock.patch.object(fs, "open_fs", new=_open_fs) + self._open_fs_mock.__enter__() + self._ftpfs_mock = mock.patch.object(fs.ftpfs, "FTPFS") + self._ftpfs_mock.__enter__() + + def tearDown(self): + self._open_fs_mock.__exit__(None, None, None) + self._ftpfs_mock.__exit__(None, None, None) + warnings.simplefilter(warnings.defaultaction) + + # doctests are not compatible with `green`, so we may want to bail out + # early if `green` is running the tests + if sys.argv[0].endswith("green"): + return tests + + # recursively traverse all library submodules and load tests from them + packages = [None, fs] + + for pkg in iter(packages.pop, None): + for (_, subpkgname, subispkg) in pkgutil.walk_packages(pkg.__path__): + # import the submodule and add it to the tests + module = importlib.import_module(".".join([pkg.__name__, subpkgname])) + + # load some useful modules / classes / mocks to the + # globals so that we don't need to explicitly import + # them in the doctests + globs = dict(**module.__dict__) + globs.update( + os=os, + fs=fs, + my_fs=_my_fs(module.__name__), + open=_open, + # NB (@althonos): This allows using OSFS in some examples, + # while not actually opening the real filesystem + OSFS=lambda path: MemoryFS(), + # NB (@althonos): This is for compatibility in `fs.registry` + print_list=lambda path: None, + ) + + # load the doctests into the unittest test suite + tests.addTests( + doctest.DocTestSuite( + module, + globs=globs, + setUp=setUp, + tearDown=tearDown, + optionflags=+doctest.ELLIPSIS, + ) + ) + + # if the submodule is a package, we need to process its submodules + # as well, so we add it to the package queue + if subispkg: + packages.append(module) + + return tests From 64ba8ce9c11e2f069f941ba3a2fd913f5f092aad Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:05:53 +0100 Subject: [PATCH 153/309] Fix docstrings of various `fs` submodules to pass the doctests --- fs/_repr.py | 2 +- fs/base.py | 7 ++--- fs/filesize.py | 6 ++--- fs/glob.py | 15 +++++------ fs/info.py | 17 +++++------- fs/opener/registry.py | 11 +++++--- fs/path.py | 7 +++-- fs/permissions.py | 2 +- fs/walk.py | 61 +++++++++++++++++++++++-------------------- fs/wrap.py | 10 +++---- 10 files changed, 71 insertions(+), 67 deletions(-) diff --git a/fs/_repr.py b/fs/_repr.py index 0a207207..d313b0a8 100644 --- a/fs/_repr.py +++ b/fs/_repr.py @@ -27,7 +27,7 @@ def make_repr(class_name, *args, **kwargs): >>> MyClass('Will') MyClass('foo', name='Will') >>> MyClass(None) - MyClass() + MyClass('foo') """ arguments = [repr(arg) for arg in args] diff --git a/fs/base.py b/fs/base.py index 7a81c5cb..fce80c64 100644 --- a/fs/base.py +++ b/fs/base.py @@ -619,7 +619,7 @@ def download(self, path, file, chunk_size=None, **options): Example: >>> with open('starwars.mov', 'wb') as write_file: - ... my_fs.download('/movies/starwars.mov', write_file) + ... my_fs.download('/Videos/starwars.mov', write_file) """ with self._lock: @@ -986,6 +986,7 @@ def lock(self): Example: >>> with my_fs.lock(): # May block ... # code here has exclusive access to the filesystem + ... pass It is a good idea to put a lock around any operations that you would like to be *atomic*. For instance if you are copying @@ -1561,9 +1562,9 @@ def match(self, patterns, name): names). Example: - >>> home_fs.match(['*.py'], '__init__.py') + >>> my_fs.match(['*.py'], '__init__.py') True - >>> home_fs.match(['*.jpg', '*.png'], 'foo.gif') + >>> my_fs.match(['*.jpg', '*.png'], 'foo.gif') False Note: diff --git a/fs/filesize.py b/fs/filesize.py index a80fd9e1..fafcc61d 100644 --- a/fs/filesize.py +++ b/fs/filesize.py @@ -61,7 +61,7 @@ def traditional(size): `str`: A string containing an abbreviated file size and units. Example: - >>> filesize.traditional(30000) + >>> fs.filesize.traditional(30000) '29.3 KB' """ @@ -87,7 +87,7 @@ def binary(size): `str`: A string containing a abbreviated file size and units. Example: - >>> filesize.binary(30000) + >>> fs.filesize.binary(30000) '29.3 KiB' """ @@ -112,7 +112,7 @@ def decimal(size): `str`: A string containing a abbreviated file size and units. Example: - >>> filesize.decimal(30000) + >>> fs.filesize.decimal(30000) '30.0 kB' """ diff --git a/fs/glob.py b/fs/glob.py index c21bb9c6..0dfa9f1c 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -172,9 +172,8 @@ def count(self): """Count files / directories / data in matched paths. Example: - >>> import fs - >>> fs.open_fs('~/projects').glob('**/*.py').count() - Counts(files=18519, directories=0, data=206690458) + >>> my_fs.glob('**/*.py').count() + Counts(files=2, directories=0, data=55) Returns: `~Counts`: A named tuple containing results. @@ -199,9 +198,8 @@ def count_lines(self): `~LineCounts`: A named tuple containing line counts. Example: - >>> import fs - >>> fs.open_fs('~/projects').glob('**/*.py').count_lines() - LineCounts(lines=5767102, non_blank=4915110) + >>> my_fs.glob('**/*.py').count_lines() + LineCounts(lines=4, non_blank=3) """ lines = 0 @@ -222,9 +220,8 @@ def remove(self): int: Number of file and directories removed. Example: - >>> import fs - >>> fs.open_fs('~/projects/my_project').glob('**/*.pyc').remove() - 29 + >>> my_fs.glob('**/*.pyc').remove() + 2 """ removes = 0 diff --git a/fs/info.py b/fs/info.py index 60a659e6..a6ec5991 100644 --- a/fs/info.py +++ b/fs/info.py @@ -106,8 +106,9 @@ def get(self, namespace, key, default=None): # noqa: F811 is not found. Example: - >>> info.get('access', 'permissions') - ['u_r', 'u_w', '_wx'] + >>> info = my_fs.getinfo("foo.py", namespaces=["details"]) + >>> info.get('details', 'type') + 2 """ try: @@ -189,12 +190,10 @@ def suffix(self): In case there is no suffix, an empty string is returned. Example: - >>> info - + >>> info = my_fs.getinfo("foo.py") >>> info.suffix '.py' - >>> info2 - + >>> info2 = my_fs.getinfo("bar") >>> info2.suffix '' @@ -211,8 +210,7 @@ def suffixes(self): """`List`: a list of any suffixes in the name. Example: - >>> info - + >>> info = my_fs.getinfo("foo.tar.gz") >>> info.suffixes ['.tar', '.gz'] @@ -228,8 +226,7 @@ def stem(self): """`str`: the name minus any suffixes. Example: - >>> info - + >>> info = my_fs.getinfo("foo.tar.gz") >>> info.stem 'foo' diff --git a/fs/opener/registry.py b/fs/opener/registry.py index f923b4ea..ff126f35 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -260,10 +260,13 @@ def manage_fs( required logic for that. Example: - >>> def print_ls(list_fs): - ... '''List a directory.''' - ... with manage_fs(list_fs) as fs: - ... print(' '.join(fs.listdir())) + The `~Registry.manage_fs` method can be used to define a small + utility function:: + + >>> def print_ls(list_fs): + ... '''List a directory.''' + ... with manage_fs(list_fs) as fs: + ... print(' '.join(fs.listdir())) This function may be used in two ways. You may either pass a ``str``, as follows:: diff --git a/fs/path.py b/fs/path.py index 9d6496b5..eab14cd1 100644 --- a/fs/path.py +++ b/fs/path.py @@ -14,6 +14,8 @@ import re import typing +import six + from .errors import IllegalBackReference if typing.TYPE_CHECKING: @@ -64,9 +66,9 @@ def normpath(path): >>> normpath("/foo//bar/frob/../baz") '/foo/bar/baz' >>> normpath("foo/../../bar") - Traceback (most recent call last) + Traceback (most recent call last): ... - IllegalBackReference: path 'foo/../../bar' contains back-references outside of filesystem" + fs.errors.IllegalBackReference: path 'foo/../../bar' contains back-references outside of filesystem """ # noqa: E501 if path in "/": @@ -86,6 +88,7 @@ def normpath(path): else: components.append(component) except IndexError: + # FIXME (@althonos): should be raised from the IndexError raise IllegalBackReference(path) return prefix + "/".join(components) diff --git a/fs/permissions.py b/fs/permissions.py index 032c3be0..3aaa6eff 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -58,7 +58,7 @@ class Permissions(object): >>> p.mode 500 >>> oct(p.mode) - '0764' + '0o764' """ diff --git a/fs/walk.py b/fs/walk.py index 0f4adb99..b4580c9b 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -146,24 +146,25 @@ def bind(cls, fs): Returns: ~fs.walk.BoundWalker: a bound walker. - Example: - >>> from fs import open_fs - >>> from fs.walk import Walker - >>> home_fs = open_fs('~/') - >>> walker = Walker.bind(home_fs) - >>> for path in walker.files(filter=['*.py']): - ... print(path) - - Unless you have written a customized walker class, you will be - unlikely to need to call this explicitly, as filesystem objects - already have a ``walk`` attribute which is a bound walker - object. + Examples: - Example: - >>> from fs import open_fs - >>> home_fs = open_fs('~/') - >>> for path in home_fs.walk.files(filter=['*.py']): - ... print(path) + Use this method to explicitly bind a filesystem instance:: + + >>> walker = Walker.bind(my_fs) + >>> for path in walker.files(filter=['*.py']): + ... print(path) + /foo.py + /bar.py + + Unless you have written a customized walker class, you will + be unlikely to need to call this explicitly, as filesystem + objects already have a ``walk`` attribute which is a bound + walker object:: + + >>> for path in my_fs.walk.files(filter=['*.py']): + ... print(path) + /foo.py + /bar.py """ return BoundWalker(fs) @@ -316,14 +317,16 @@ def walk( `~fs.info.Info` objects for directories and files in ````. Example: - >>> home_fs = open_fs('~/') >>> walker = Walker(filter=['*.py']) - >>> namespaces = ['details'] - >>> for path, dirs, files in walker.walk(home_fs, namespaces) + >>> for path, dirs, files in walker.walk(my_fs, namespaces=["details"]): ... print("[{}]".format(path)) ... print("{} directories".format(len(dirs))) ... total = sum(info.size for info in files) - ... print("{} bytes {}".format(total)) + ... print("{} bytes".format(total)) + [/] + 2 directories + 55 bytes + ... """ _path = abspath(normpath(path)) @@ -495,10 +498,9 @@ class BoundWalker(typing.Generic[_F]): `BoundWalker` object. Example: - >>> import fs - >>> home_fs = fs.open_fs('~/') - >>> home_fs.walk - BoundWalker(OSFS('/Users/will', encoding='utf-8')) + >>> tmp_fs = fs.tempfs.TempFS() + >>> tmp_fs.walk + BoundWalker(TempFS()) A `BoundWalker` is callable. Calling it is an alias for the `~fs.walk.BoundWalker.walk` method. @@ -575,13 +577,16 @@ def walk( `~fs.info.Info` objects for directories and files in ````. Example: - >>> home_fs = open_fs('~/') >>> walker = Walker(filter=['*.py']) - >>> for path, dirs, files in walker.walk(home_fs, namespaces=['details']): + >>> for path, dirs, files in walker.walk(my_fs, namespaces=['details']): ... print("[{}]".format(path)) ... print("{} directories".format(len(dirs))) ... total = sum(info.size for info in files) - ... print("{} bytes {}".format(total)) + ... print("{} bytes".format(total)) + [/] + 2 directories + 55 bytes + ... This method invokes `Walker.walk` with bound `FS` object. diff --git a/fs/wrap.py b/fs/wrap.py index 3ae4aa9f..e03ef805 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -2,14 +2,12 @@ Here's an example that opens a filesystem then makes it *read only*:: - >>> from fs import open_fs - >>> from fs.wrap import read_only - >>> projects_fs = open_fs('~/projects') - >>> read_only_projects_fs = read_only(projects_fs) - >>> read_only_projects_fs.remove('__init__.py') + >>> home_fs = fs.open_fs('~') + >>> read_only_home_fs = fs.wrap.read_only(home_fs) + >>> read_only_home_fs.removedir('Desktop') Traceback (most recent call last): ... - fs.errors.ResourceReadOnly: resource '__init__.py' is read only + fs.errors.ResourceReadOnly: resource 'Desktop' is read only """ From b95adcb3d246a1a680042223844c136da53c246b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:06:23 +0100 Subject: [PATCH 154/309] Add some more examples opening a FTPFS in `fs.ftpfs` module --- fs/ftpfs.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index c5929ee1..925af8af 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -363,17 +363,21 @@ class FTPFS(FS): Create with the constructor:: >>> from fs.ftpfs import FTPFS - >>> ftp_fs = FTPFS() + >>> ftp_fs = FTPFS("demo.wftpserver.com") Or via an FS URL:: - >>> import fs - >>> ftp_fs = fs.open_fs('ftp://') + >>> ftp_fs = fs.open_fs('ftp://test.rebex.net') Or via an FS URL, using TLS:: - >>> import fs - >>> ftp_fs = fs.open_fs('ftps://') + >>> ftp_fs = fs.open_fs('ftps://demo.wftpserver.com') + + You can also use a non-anonymous username, and optionally a + password, even within a FS URL:: + + >>> ftp_fs = FTPFS("test.rebex.net", user="demo", passwd="password") + >>> ftp_fs = fs.open_fs('ftp://demo:password@test.rebex.net') """ From 69fb190436ef30653cd75dff2a770ebc14000caa Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:06:50 +0100 Subject: [PATCH 155/309] Add `TempFS.close` docstring with a warning about the `auto_clean` parameter --- fs/tempfs.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fs/tempfs.py b/fs/tempfs.py index a1e5a3d2..469d7dc3 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -71,6 +71,28 @@ def __str__(self): def close(self): # type: () -> None + """Close the filesystem and release any resources. + + It is important to call this method when you have finished + working with the filesystem. Some filesystems may not finalize + changes until they are closed (archives for example). You may + call this method explicitly (it is safe to call close multiple + times), or you can use the filesystem as a context manager to + automatically close. + + Hint: + Depending on the value of ``auto_clean`` passed when creating + the `TempFS`, the underlying temporary folder may be removed + or not. + + Example: + >>> tmp_fs = TempFS(auto_clean=False) + >>> syspath = tmp_fs.getsyspath("/") + >>> tmp_fs.close() + >>> os.path.exists(syspath) + True + + """ if self._auto_clean: self.clean() super(TempFS, self).close() From 9291f7836cb549d109e7f21721e41ecf7baac766 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:15:24 +0100 Subject: [PATCH 156/309] Remove unused imports in `fs.path` and `tests.test_doctest` --- fs/path.py | 2 -- tests/test_doctest.py | 1 - 2 files changed, 3 deletions(-) diff --git a/fs/path.py b/fs/path.py index eab14cd1..13641be1 100644 --- a/fs/path.py +++ b/fs/path.py @@ -14,8 +14,6 @@ import re import typing -import six - from .errors import IllegalBackReference if typing.TYPE_CHECKING: diff --git a/tests/test_doctest.py b/tests/test_doctest.py index c96cb84b..de8f5789 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -21,7 +21,6 @@ import fs.opener.parse from fs.memoryfs import MemoryFS from fs.subfs import ClosingSubFS -from fs.tempfs import TempFS # --- Mocks ------------------------------------------------------------------ From 8388a02f5a292700f87542ca14ae916fa6a6549c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 18:16:30 +0100 Subject: [PATCH 157/309] Remove empty line from `fs.walk.Walk.bind` docstring --- fs/walk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/walk.py b/fs/walk.py index b4580c9b..f539fa9d 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -147,7 +147,6 @@ def bind(cls, fs): ~fs.walk.BoundWalker: a bound walker. Examples: - Use this method to explicitly bind a filesystem instance:: >>> walker = Walker.bind(my_fs) From 34040619d3ee7f3e4f49d9c9c5b281ab11d9270e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 19:23:09 +0100 Subject: [PATCH 158/309] Add a workaround for `pytest` not supporting the `load_tests` protocol --- tests/test_doctest.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/test_doctest.py b/tests/test_doctest.py index de8f5789..01d2ad2c 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -9,6 +9,7 @@ import types import warnings import tempfile +import unittest try: from unittest import mock @@ -107,7 +108,7 @@ def _load_tests_from_module(tests, module, globs, setUp=None, tearDown=None): return tests -def load_tests(loader, tests, ignore): +def _load_tests(loader, tests, ignore): """`load_test` function used by unittest to find the doctests.""" # NB (@althonos): we only test docstrings on Python 3 because it's @@ -135,7 +136,6 @@ def tearDown(self): # recursively traverse all library submodules and load tests from them packages = [None, fs] - for pkg in iter(packages.pop, None): for (_, subpkgname, subispkg) in pkgutil.walk_packages(pkg.__path__): # import the submodule and add it to the tests @@ -174,3 +174,30 @@ def tearDown(self): packages.append(module) return tests + + +# --- Unit test wrapper ------------------------------------------------------ +# +# NB (@althonos): Since pytest doesn't support the `load_tests` protocol +# above, we manually build a `unittest.TestCase` using a dedicated test +# method for each doctest. This should be safe to remove when pytest +# supports it, or if we move away from pytest to run tests. + + +class TestDoctest(unittest.TestCase): + pass + + +def make_wrapper(x): + def _test_wrapper(self): + x.setUp() + try: + x.runTest() + finally: + x.tearDown() + + return _test_wrapper + + +for x in _load_tests(None, unittest.TestSuite(), False): + setattr(TestDoctest, "test_{}".format(x.id().replace(".", "_")), make_wrapper(x)) From 3ea23b592a61b0f117d25ba16876393bd8a83711 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 21:25:15 +0100 Subject: [PATCH 159/309] Fix type annotation and docs of `temp_fs` parameter of archive FS --- fs/tarfs.py | 17 +++++++++-------- fs/zipfs.py | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/fs/tarfs.py b/fs/tarfs.py index 85e74840..0e808fe1 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -66,10 +66,10 @@ def _get_member_info(member, encoding): class TarFS(WrapFS): """Read and write tar files. - There are two ways to open a TarFS for the use cases of reading + There are two ways to open a `TarFS` for the use cases of reading a tar file, and creating a new one. - If you open the TarFS with ``write`` set to `False` (the + If you open the `TarFS` with ``write`` set to `False` (the default), then the filesystem will be a read only filesystem which maps to the files and directories within the tar file. Files are decompressed on the fly when you open them. @@ -79,9 +79,9 @@ class TarFS(WrapFS): with TarFS('foo.tar.gz') as tar_fs: readme = tar_fs.readtext('readme.txt') - If you open the TarFS with ``write`` set to `True`, then the TarFS + If you open the TarFS with ``write`` set to `True`, then the `TarFS` will be a empty temporary filesystem. Any files / directories you - create in the TarFS will be written in to a tar file when the TarFS + create in the `TarFS` will be written in to a tar file when the `TarFS` is closed. The compression is set from the new file name but may be set manually with the ``compression`` argument. @@ -100,8 +100,9 @@ class TarFS(WrapFS): use default (`False`) to read an existing tar file. compression (str, optional): Compression to use (one of the formats supported by `tarfile`: ``xz``, ``gz``, ``bz2``, or `None`). - temp_fs (str): An FS URL for the temporary filesystem - used to store data prior to tarring. + temp_fs (str): An FS URL or an FS instance to use to store + data prior to tarring. Defaults to creating a new + `~fs.tempfs.TempFS`. """ @@ -118,7 +119,7 @@ def __new__( # type: ignore write=False, # type: bool compression=None, # type: Optional[Text] encoding="utf-8", # type: Text - temp_fs="temp://__tartemp__", # type: Text + temp_fs="temp://__tartemp__", # type: Union[Text, FS] ): # type: (...) -> FS if isinstance(file, (six.text_type, six.binary_type)): @@ -164,7 +165,7 @@ def __init__( file, # type: Union[Text, BinaryIO] compression=None, # type: Optional[Text] encoding="utf-8", # type: Text - temp_fs="temp://__tartemp__", # type: Text + temp_fs="temp://__tartemp__", # type: Union[Text, FS] ): # noqa: D107 # type: (...) -> None self._file = file # type: Union[Text, BinaryIO] diff --git a/fs/zipfs.py b/fs/zipfs.py index d8300a26..12d8a668 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -124,11 +124,11 @@ def tell(self): class ZipFS(WrapFS): """Read and write zip files. - There are two ways to open a ZipFS for the use cases of reading + There are two ways to open a `ZipFS` for the use cases of reading a zip file, and creating a new one. - If you open the ZipFS with ``write`` set to `False` (the default) - then the filesystem will be a read only filesystem which maps to + If you open the `ZipFS` with ``write`` set to `False` (the default) + then the filesystem will be a read-only filesystem which maps to the files and directories within the zip file. Files are decompressed on the fly when you open them. @@ -137,12 +137,12 @@ class ZipFS(WrapFS): with ZipFS('foo.zip') as zip_fs: readme = zip_fs.readtext('readme.txt') - If you open the ZipFS with ``write`` set to `True`, then the ZipFS - will be a empty temporary filesystem. Any files / directories you - create in the ZipFS will be written in to a zip file when the ZipFS + If you open the `ZipFS` with ``write`` set to `True`, then the `ZipFS` + will be an empty temporary filesystem. Any files / directories you + create in the `ZipFS` will be written in to a zip file when the `ZipFS` is closed. - Here's how you might write a new zip file containing a readme.txt + Here's how you might write a new zip file containing a ``readme.txt`` file:: with ZipFS('foo.zip', write=True) as new_zip: @@ -158,8 +158,9 @@ class ZipFS(WrapFS): (default) to read an existing zip file. compression (int): Compression to use (one of the constants defined in the `zipfile` module in the stdlib). - temp_fs (str): An FS URL for the temporary filesystem used to - store data prior to zipping. + temp_fs (str or FS): An FS URL or an FS instance to use to + store data prior to zipping. Defaults to creating a new + `~fs.tempfs.TempFS`. """ @@ -170,7 +171,7 @@ def __new__( # type: ignore write=False, # type: bool compression=zipfile.ZIP_DEFLATED, # type: int encoding="utf-8", # type: Text - temp_fs="temp://__ziptemp__", # type: Text + temp_fs="temp://__ziptemp__", # type: Union[Text, FS] ): # type: (...) -> FS # This magic returns a different class instance based on the @@ -205,7 +206,7 @@ def __init__( file, # type: Union[Text, BinaryIO] compression=zipfile.ZIP_DEFLATED, # type: int encoding="utf-8", # type: Text - temp_fs="temp://__ziptemp__", # type: Text + temp_fs="temp://__ziptemp__", # type: Union[Text, FS] ): # noqa: D107 # type: (...) -> None self._file = file From f640b9cda85c75ea7a04ce4109664bcbf9542e55 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 22:04:27 +0100 Subject: [PATCH 160/309] Document some exceptions expected on edge cases of `fs.base.FS` methods --- fs/base.py | 43 +++++++++++++++++++++++++++++++------------ fs/ftpfs.py | 1 - 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/fs/base.py b/fs/base.py index fce80c64..0a74fa18 100644 --- a/fs/base.py +++ b/fs/base.py @@ -152,12 +152,16 @@ def getinfo(self, path, namespaces=None): Arguments: path (str): A path to a resource on the filesystem. - namespaces (list, optional): Info namespaces to query - (defaults to *[basic]*). + namespaces (list, optional): Info namespaces to query. The + `"basic"` namespace is alway included in the returned + info, whatever the value of `namespaces` may be. Returns: ~fs.info.Info: resource information object. + Raises: + fs.errors.ResourceNotFound: If ``path`` does not exist. + For more information regarding resource information, see :ref:`info`. """ @@ -235,10 +239,12 @@ def openbin( io.IOBase: a *file-like* object. Raises: - fs.errors.FileExpected: If the path is not a file. - fs.errors.FileExists: If the file exists, and *exclusive mode* - is specified (``x`` in the mode). - fs.errors.ResourceNotFound: If the path does not exist. + fs.errors.FileExpected: If ``path`` exists and is not a file. + fs.errors.FileExists: If the ``path`` exists, and + *exclusive mode* is specified (``x`` in the mode). + fs.errors.ResourceNotFound: If ``path`` does not exist and + ``mode`` does not imply creating the file, or if any + ancestor of ``path`` does not exist. """ @@ -402,6 +408,7 @@ def copy(self, src_path, dst_path, overwrite=False): and ``overwrite`` is `False`. fs.errors.ResourceNotFound: If a parent directory of ``dst_path`` does not exist. + fs.errors.FileExpected: If ``src_path`` is not a file. """ with self._lock: @@ -587,6 +594,7 @@ def readbytes(self, path): bytes: the file contents. Raises: + fs.errors.FileExpected: if ``path`` exists but is not a file. fs.errors.ResourceNotFound: if ``path`` does not exist. """ @@ -603,6 +611,10 @@ def download(self, path, file, chunk_size=None, **options): This may be more efficient that opening and copying files manually if the filesystem supplies an optimized method. + Note that the file object ``file`` will *not* be closed by this + method. Take care to close it after this method completes + (ideally with a context manager). + Arguments: path (str): Path to a resource. file (file-like): A file-like object open for writing in @@ -613,14 +625,13 @@ def download(self, path, file, chunk_size=None, **options): **options: Implementation specific options required to open the source file. - Note that the file object ``file`` will *not* be closed by this - method. Take care to close it after this method completes - (ideally with a context manager). - Example: >>> with open('starwars.mov', 'wb') as write_file: ... my_fs.download('/Videos/starwars.mov', write_file) + Raises: + fs.errors.ResourceNotFound: if ``path`` does not exist. + """ with self._lock: with self.openbin(path, **options) as read_file: @@ -698,7 +709,7 @@ def getmeta(self, namespace="standard"): network. read_only `True` if this filesystem is read only. supports_rename `True` if this filesystem supports an - `os.rename` operation. + `os.rename` operation (or equivalent). =================== ============================================ Most builtin filesystems will provide all these keys, and third- @@ -726,6 +737,9 @@ def getsize(self, path): Returns: int: the *size* of the resource. + Raises: + fs.errors.ResourceNotFound: if ``path`` does not exist. + The *size* of a file is the total number of readable bytes, which may not reflect the exact number of bytes of reserved disk space (or other storage medium). @@ -1018,6 +1032,8 @@ def movedir(self, src_path, dst_path, create=False): Raises: fs.errors.ResourceNotFound: if ``dst_path`` does not exist, and ``create`` is `False`. + fs.errors.DirectoryExpected: if ``src_path`` or one of its + ancestors is not a directory. """ with self._lock: @@ -1617,13 +1633,16 @@ def hash(self, path, name): Arguments: path(str): A path on the filesystem. name(str): - One of the algorithms supported by the hashlib module, e.g. `"md5"` + One of the algorithms supported by the `hashlib` module, + e.g. `"md5"` or `"sha256"`. Returns: str: The hex digest of the hash. Raises: fs.errors.UnsupportedHash: If the requested hash is not supported. + fs.errors.ResourceNotFound: If ``path`` does not exist. + fs.errors.FileExpected: If ``path`` exists but is not a file. """ self.validatepath(path) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 925af8af..35e39ca8 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -358,7 +358,6 @@ class FTPFS(FS): FTPS, or FTP Secure. TLS will be enabled when using the ftps:// protocol, or when setting the `tls` argument to True in the constructor. - Examples: Create with the constructor:: From 61db5b0863ef6eae3bf4d3650d1d93705481c7ed Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 23:15:38 +0100 Subject: [PATCH 161/309] Ignore type-checking branches when measuring coverage --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index a5c46904..fcc75584 100644 --- a/setup.cfg +++ b/setup.cfg @@ -119,6 +119,7 @@ skip_covered = true exclude_lines = pragma: no cover if False: + it typing.TYPE_CHECKING: @typing.overload @overload From b5cc24f8a041947ea8f895a2c2226045285cc3c6 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 23:36:39 +0100 Subject: [PATCH 162/309] Add overloaded annotations of `epoch_to_datetime` to `fs.time` --- fs/time.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/fs/time.py b/fs/time.py index f1638aa3..cdf06061 100644 --- a/fs/time.py +++ b/fs/time.py @@ -4,10 +4,14 @@ from __future__ import print_function from __future__ import unicode_literals +import typing from calendar import timegm from datetime import datetime from pytz import UTC, timezone +if typing.TYPE_CHECKING: + from typing import Optional + utcfromtimestamp = datetime.utcfromtimestamp utclocalize = UTC.localize @@ -20,7 +24,19 @@ def datetime_to_epoch(d): return timegm(d.utctimetuple()) -def epoch_to_datetime(t): +@typing.overload +def epoch_to_datetime(t): # noqa: D103 + # type: (None) -> None + pass + + +@typing.overload +def epoch_to_datetime(t): # noqa: D103 # type: (int) -> datetime + pass + + +def epoch_to_datetime(t): + # type: (Optional[int]) -> Optional[datetime] """Convert epoch time to a UTC datetime.""" return utclocalize(utcfromtimestamp(t)) if t is not None else None From 5bc93ef1177cb39d21c5c1ebb801232119ee86d1 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 25 Mar 2021 23:47:11 +0100 Subject: [PATCH 163/309] Remove compatibility code from `tests.test_doctest` --- tests/test_doctest.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test_doctest.py b/tests/test_doctest.py index 01d2ad2c..ab57fe9b 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -59,18 +59,15 @@ def _open_fs(path): def _my_fs(module): """Create a mock filesystem to be used in examples.""" my_fs = MemoryFS() - if module == "fs.base": my_fs.makedir("Desktop") my_fs.makedir("Videos") my_fs.touch("Videos/starwars.mov") my_fs.touch("file.txt") - elif module == "fs.info": my_fs.touch("foo.tar.gz") my_fs.settext("foo.py", "print('Hello, world!')") my_fs.makedir("bar") - elif module in {"fs.walk", "fs.glob"}: my_fs.makedir("dir1") my_fs.makedir("dir2") @@ -78,10 +75,6 @@ def _my_fs(module): my_fs.touch("foo.pyc") my_fs.settext("bar.py", "print('ok')\n\n# this is a comment\n") my_fs.touch("bar.pyc") - - # # used in `fs.glob` - # home_fs.touch("foo.pyc") - # home_fs.touch("bar.pyc") return my_fs @@ -129,11 +122,6 @@ def tearDown(self): self._ftpfs_mock.__exit__(None, None, None) warnings.simplefilter(warnings.defaultaction) - # doctests are not compatible with `green`, so we may want to bail out - # early if `green` is running the tests - if sys.argv[0].endswith("green"): - return tests - # recursively traverse all library submodules and load tests from them packages = [None, fs] for pkg in iter(packages.pop, None): From 902362d693b8c4b93c1ec00e9e2b706592c9055a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 17:09:08 +0100 Subject: [PATCH 164/309] Add some examples showing how to build a `TempFS` --- fs/tempfs.py | 28 ++++++++++++++++++++++++++-- tests/test_doctest.py | 3 +-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/fs/tempfs.py b/fs/tempfs.py index 469d7dc3..5fdc2f61 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -27,7 +27,31 @@ @six.python_2_unicode_compatible class TempFS(OSFS): - """A temporary filesystem on the OS.""" + """A temporary filesystem on the OS. + + Temporary filesystems are created using the `tempfile.mkdtemp` + function to obtain a temporary folder in an OS-specific location. + You can provide an alternative location with the ``temp_dir`` + argument of the constructor. + + Examples: + Create with the constructor:: + + >>> from fs.tempfs import TempFS + >>> tmp_fs = TempFS() + + Or via an FS URL:: + + >>> import fs + >>> tmp_fs = fs.open_fs("temp://") + + Use a specific identifier for the temporary folder to better + illustrate its purpose:: + + >>> named_tmp_fs = fs.open_fs("temp://local_copy") + >>> named_tmp_fs = TempFS(identifier="local_copy") + + """ def __init__( self, @@ -43,7 +67,7 @@ def __init__( identifier (str): A string to distinguish the directory within the OS temp location, used as part of the directory name. temp_dir (str, optional): An OS path to your temp directory - (leave as `None` to auto-detect) + (leave as `None` to auto-detect). auto_clean (bool): If `True` (the default), the directory contents will be wiped on close. ignore_clean_errors (bool): If `True` (the default), any errors diff --git a/tests/test_doctest.py b/tests/test_doctest.py index ab57fe9b..4465eca2 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -5,7 +5,6 @@ import importlib import os import pkgutil -import sys import types import warnings import tempfile @@ -50,7 +49,7 @@ def _open_fs(path): if parse_result.path is not None: home_fs = home_fs.opendir(parse_result.path, factory=ClosingSubFS) return home_fs - elif parse_result.protocol in {"ftp", "ftps", "mem"}: + elif parse_result.protocol in {"ftp", "ftps", "mem", "temp"}: return MemoryFS() else: raise RuntimeError("not allowed in doctests: {}".format(path)) From 9a576d8266076f01ccde7e1c4562193c811d8bd5 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 18:05:46 +0100 Subject: [PATCH 165/309] Rewrite docstrings of some private helpers in `doctest` format so they are tested --- fs/_ftp_parse.py | 35 +++++++++++++++++++++++++++-------- fs/_url_tools.py | 30 +++++++++++++++++------------- fs/base.py | 2 +- fs/opener/registry.py | 2 +- tests/test_doctest.py | 4 ++++ 5 files changed, 50 insertions(+), 23 deletions(-) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index a9088ab4..defc55ee 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -56,9 +56,7 @@ def get_decoders(): - """ - Returns all available FTP LIST line decoders with their matching regexes. - """ + """Return all available FTP LIST line decoders with their matching regexes.""" decoders = [ (RE_LINUX, decode_linux), (RE_WINDOWSNT, decode_windowsnt), @@ -149,13 +147,34 @@ def _decode_windowsnt_time(mtime): def decode_windowsnt(line, match): - """ - Decodes a Windows NT FTP LIST line like one of these: + """Decode a Windows NT FTP LIST line. + + Examples: + Decode a directory line:: + + >>> line = "11-02-18 02:12PM images" + >>> match = RE_WINDOWSNT.match(line) + >>> pprint(decode_windowsnt(line, match)) + {'basic': {'is_dir': True, 'name': 'images'}, + 'details': {'modified': 1518358320.0, 'type': 1}, + 'ftp': {'ls': '11-02-18 02:12PM images'}} + + Decode a file line:: + + >>> line = "11-02-18 03:33PM 9276 logo.gif" + >>> match = RE_WINDOWSNT.match(line) + >>> pprint(decode_windowsnt(line, match)) + {'basic': {'is_dir': False, 'name': 'logo.gif'}, + 'details': {'modified': 1518363180.0, 'size': 9276, 'type': 2}, + 'ftp': {'ls': '11-02-18 03:33PM 9276 logo.gif'}} + + Alternatively, the time might also be present in 24-hour format:: - `11-02-18 02:12PM images` - `11-02-18 03:33PM 9276 logo.gif` + >>> line = "11-02-18 15:33 9276 logo.gif" + >>> match = RE_WINDOWSNT.match(line) + >>> decode_windowsnt(line, match)["details"]["modified"] + 1518363180.0 - Alternatively, the time (02:12PM) might also be present in 24-hour format (14:12). """ is_dir = match.group("size") == "" diff --git a/fs/_url_tools.py b/fs/_url_tools.py index 64c58bd6..af55ff74 100644 --- a/fs/_url_tools.py +++ b/fs/_url_tools.py @@ -11,13 +11,15 @@ def url_quote(path_snippet): # type: (Text) -> Text - """ - On Windows, it will separate drive letter and quote windows - path alone. No magic on Unix-alie path, just pythonic - `pathname2url` + """Quote a URL without quoting the Windows drive letter, if any. + + On Windows, it will separate drive letter and quote Windows + path alone. No magic on Unix-like path, just pythonic + `~urllib.request.pathname2url`. Arguments: - path_snippet: a file path, relative or absolute. + path_snippet (str): a file path, relative or absolute. + """ if _WINDOWS_PLATFORM and _has_drive_letter(path_snippet): drive_letter, path = path_snippet.split(":", 1) @@ -34,17 +36,19 @@ def url_quote(path_snippet): def _has_drive_letter(path_snippet): # type: (Text) -> bool - """ - The following path will get True - D:/Data - C:\\My Dcouments\\ test + """Check whether a path contains a drive letter. - And will get False + Arguments: + path_snippet (str): a file path, relative or absolute. - /tmp/abc:test + Example: + >>> _has_drive_letter("D:/Data") + True + >>> _has_drive_letter(r"C:\\System32\\ test") + True + >>> _has_drive_letter("/tmp/abc:test") + False - Arguments: - path_snippet: a file path, relative or absolute. """ windows_drive_pattern = ".:[/\\\\].*$" return re.match(windows_drive_pattern, path_snippet) is not None diff --git a/fs/base.py b/fs/base.py index 0a74fa18..b727f998 100644 --- a/fs/base.py +++ b/fs/base.py @@ -709,7 +709,7 @@ def getmeta(self, namespace="standard"): network. read_only `True` if this filesystem is read only. supports_rename `True` if this filesystem supports an - `os.rename` operation (or equivalent). + `os.rename` operation. =================== ============================================ Most builtin filesystems will provide all these keys, and third- diff --git a/fs/opener/registry.py b/fs/opener/registry.py index ff126f35..54e2dda1 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -217,7 +217,7 @@ def open_fs( filesystem *needs* to be writable, which is relevant for some archive filesystems. Passing ``writeable=False`` will **not** make the return filesystem read-only. For this, - consider using `fs.wrap.WrapReadOnly` to wrap the returned + consider using `fs.wrap.read_only` to wrap the returned instance. """ diff --git a/tests/test_doctest.py b/tests/test_doctest.py index 4465eca2..22c02357 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -8,7 +8,9 @@ import types import warnings import tempfile +import time import unittest +from pprint import pprint try: from unittest import mock @@ -142,6 +144,8 @@ def tearDown(self): OSFS=lambda path: MemoryFS(), # NB (@althonos): This is for compatibility in `fs.registry` print_list=lambda path: None, + pprint=pprint, + time=time, ) # load the doctests into the unittest test suite From 7dbeb88b0d992bf29ea20552c2116d14458f921b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Fri, 26 Mar 2021 18:06:38 +0100 Subject: [PATCH 166/309] Improve `Info.is_writable` to document the `_write` key of every namespace --- fs/info.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/fs/info.py b/fs/info.py index a6ec5991..03bf27cd 100644 --- a/fs/info.py +++ b/fs/info.py @@ -41,7 +41,7 @@ class Info(object): raw_info (dict): A dict containing resource info. to_datetime (callable): A callable that converts an epoch time to a datetime object. The default uses - :func:`~fs.time.epoch_to_datetime`. + `~fs.time.epoch_to_datetime`. """ @@ -132,7 +132,8 @@ def is_writeable(self, namespace, key): # type: (Text, Text) -> bool """Check if a given key in a namespace is writable. - Uses `~fs.base.FS.setinfo`. + When creating an `Info` object, you can add a ``_write`` key to + each raw namespace that lists which keys are writable or not. Arguments: namespace (str): A namespace identifier. @@ -141,6 +142,24 @@ def is_writeable(self, namespace, key): Returns: bool: `True` if the key can be modified, `False` otherwise. + Example: + Create an `Info` object that marks only the ``modified`` key + as writable in the ``details`` namespace:: + + >>> now = time.time() + >>> info = Info({ + ... "basic": {"name": "foo", "is_dir": False}, + ... "details": { + ... "modified": now, + ... "created": now, + ... "_write": ["modified"], + ... } + ... }) + >>> info.is_writeable("details", "created") + False + >>> info.is_writeable("details", "modified") + True + """ _writeable = self.get(namespace, "_write", ()) return key in _writeable From 55d2b374a904640968681e582dbe375940063225 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 14:53:14 +0100 Subject: [PATCH 167/309] Standardize behaviour of `removetree("/")` and add test case to `FSTestCase` --- fs/base.py | 30 +++++++++++++++++++++++++++--- fs/test.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/fs/base.py b/fs/base.py index b727f998..79cf2fb8 100644 --- a/fs/base.py +++ b/fs/base.py @@ -273,7 +273,7 @@ def removedir(self, path): Raises: fs.errors.DirectoryNotEmpty: If the directory is not empty ( see `~fs.base.FS.removetree` for a way to remove the - directory contents.). + directory contents). fs.errors.DirectoryExpected: If the path does not refer to a directory. fs.errors.ResourceNotFound: If no resource exists at the @@ -1215,14 +1215,38 @@ def opendir( def removetree(self, dir_path): # type: (Text) -> None - """Recursively remove the contents of a directory. + """Recursively remove a directory and all its contents. - This method is similar to `~fs.base.removedir`, but will + This method is similar to `~fs.base.FS.removedir`, but will remove the contents of the directory if it is not empty. Arguments: dir_path (str): Path to a directory on the filesystem. + Caution: + A filesystem should never delete its root folder, so + ``FS.removetree("/")`` has different semantics: the + contents of the root folder will be deleted, but the + root will be untouched:: + + >>> home_fs = fs.open_fs("~") + >>> home_fs.removetree("/") + >>> home_fs.exists("/") + True + >>> home_fs.isempty("/") + True + + Combined with `~fs.base.FS.opendir`, this can be used + to clear a directory without removing the directory + itself:: + + >>> home_fs = fs.open_fs("~") + >>> home_fs.opendir("/Videos").removetree("/") + >>> home_fs.exists("/Videos") + True + >>> home_fs.isempty("/Videos") + True + """ _dir_path = abspath(normpath(dir_path)) with self._lock: diff --git a/fs/test.py b/fs/test.py index 4d4e4518..a717974d 100644 --- a/fs/test.py +++ b/fs/test.py @@ -292,6 +292,15 @@ def assert_not_exists(self, path): """ self.assertFalse(self.fs.exists(path)) + def assert_isempty(self, path): + """Assert a path is an empty directory. + + Arguments: + path (str): A path on the filesystem. + + """ + self.assertTrue(self.fs.isempty(path)) + def assert_isfile(self, path): """Assert a path is a file. @@ -1101,6 +1110,7 @@ def test_removedir(self): self.fs.removedir("foo/bar") def test_removetree(self): + self.fs.makedirs("spam") self.fs.makedirs("foo/bar/baz") self.fs.makedirs("foo/egg") self.fs.makedirs("foo/a/b/c/d/e") @@ -1116,6 +1126,7 @@ def test_removetree(self): self.fs.removetree("foo") self.assert_not_exists("foo") + self.assert_exists("spam") # Errors on files self.fs.create("bar") @@ -1126,6 +1137,34 @@ def test_removetree(self): with self.assertRaises(errors.ResourceNotFound): self.fs.removetree("foofoo") + def test_removetree_root(self): + self.fs.makedirs("foo/bar/baz") + self.fs.makedirs("foo/egg") + self.fs.makedirs("foo/a/b/c/d/e") + self.fs.create("foo/egg.txt") + self.fs.create("foo/bar/egg.bin") + self.fs.create("foo/a/b/c/1.txt") + self.fs.create("foo/a/b/c/2.txt") + self.fs.create("foo/a/b/c/3.txt") + + self.assert_exists("foo/egg.txt") + self.assert_exists("foo/bar/egg.bin") + + # removetree("/") removes the contents, + # but not the root folder itself + self.fs.removetree("/") + self.assert_exists("/") + self.assert_isempty("/") + + # we check we can create a file after + # to catch potential issues with the + # root folder being deleted on faulty + # implementations + self.fs.create("egg") + self.fs.makedir("yolk") + self.assert_exists("egg") + self.assert_exists("yolk") + def test_setinfo(self): self.fs.create("birthday.txt") now = math.floor(time.time()) From 97056deabad0c41d13c7dfbf2d122e21c7687df0 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 14:53:43 +0100 Subject: [PATCH 168/309] Fix `WrapFS.removetree` so that it respects the root path semantics --- fs/wrapfs.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index e40a7a83..5944b111 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -12,7 +12,7 @@ from .copy import copy_file, copy_dir from .info import Info from .move import move_file, move_dir -from .path import abspath, normpath +from .path import abspath, join, normpath from .error_tools import unwrap_errors if typing.TYPE_CHECKING: @@ -217,11 +217,20 @@ def removetree(self, dir_path): # type: (Text) -> None self.check() _path = abspath(normpath(dir_path)) - if _path == "/": - raise errors.RemoveRootError() - _fs, _path = self.delegate_path(dir_path) + _delegate_fs, _delegate_path = self.delegate_path(dir_path) with unwrap_errors(dir_path): - _fs.removetree(_path) + if _path == "/": + # with root path, we must remove the contents but + # not the directory itself, so we can't just directly + # delegate + for info in _delegate_fs.scandir(_delegate_path): + info_path = join(_delegate_path, info.name) + if info.is_dir: + _delegate_fs.removetree(info_path) + else: + _delegate_fs.remove(info_path) + else: + _delegate_fs.removetree(_delegate_path) def scandir( self, From 65ad3f7707b46e767b302a3021065ccf708cb246 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 15:14:21 +0100 Subject: [PATCH 169/309] Explicitly check the *basic* namespace is always returned by `FS.getinfo` --- fs/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/test.py b/fs/test.py index a717974d..e40e43cf 100644 --- a/fs/test.py +++ b/fs/test.py @@ -466,6 +466,7 @@ def test_getinfo(self): root_info = self.fs.getinfo("/") self.assertEqual(root_info.name, "") self.assertTrue(root_info.is_dir) + self.assertIn("basic", root_info.namespaces) # Make a file of known size self.fs.writebytes("foo", b"bar") @@ -473,17 +474,20 @@ def test_getinfo(self): # Check basic namespace info = self.fs.getinfo("foo").raw + self.assertIn("basic", info) self.assertIsInstance(info["basic"]["name"], text_type) self.assertEqual(info["basic"]["name"], "foo") self.assertFalse(info["basic"]["is_dir"]) # Check basic namespace dir info = self.fs.getinfo("dir").raw + self.assertIn("basic", info) self.assertEqual(info["basic"]["name"], "dir") self.assertTrue(info["basic"]["is_dir"]) # Get the info info = self.fs.getinfo("foo", namespaces=["details"]).raw + self.assertIn("basic", info) self.assertIsInstance(info, dict) self.assertEqual(info["details"]["size"], 3) self.assertEqual(info["details"]["type"], int(ResourceType.file)) From 53f6e14dc58eebd9758fc73129e4fb99d13adff5 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 15:15:33 +0100 Subject: [PATCH 170/309] Deprecate `FS.getbasic` since it is superseded by `FS.getinfo` --- fs/base.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index 79cf2fb8..59b70b61 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1209,7 +1209,7 @@ def opendir( _factory = factory or self.subfs_class or SubFS - if not self.getbasic(path).is_dir: + if not self.getinfo(path).is_dir: raise errors.DirectoryExpected(path=path) return _factory(self, path) @@ -1553,7 +1553,16 @@ def getbasic(self, path): Returns: ~fs.info.Info: Resource information object for ``path``. + Note: + .. deprecated:: 2.4.13 + Please use `~FS.getinfo` directly, which is + required to always return the *basic* namespace. + """ + warnings.warn( + "method 'getbasic' has been deprecated, please use 'getinfo'", + DeprecationWarning, + ) return self.getinfo(path, namespaces=["basic"]) def getdetails(self, path): From 7fe529b3a881b029bd04198351407f93aa9f5fbe Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 15:36:24 +0100 Subject: [PATCH 171/309] Document more of the expected exceptions in `fs.base.FS` interface --- fs/base.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/fs/base.py b/fs/base.py index 59b70b61..4966d0ca 100644 --- a/fs/base.py +++ b/fs/base.py @@ -431,6 +431,8 @@ def copydir(self, src_path, dst_path, create=False): Raises: fs.errors.ResourceNotFound: If the ``dst_path`` does not exist, and ``create`` is not `True`. + fs.errors.DirectoryExpected: If ``src_path`` is not a + directory. """ with self._lock: @@ -474,6 +476,9 @@ def desc(self, path): Returns: str: a short description of the path. + Raises: + fs.errors.ResourceNotFound: If ``path`` does not exist. + """ if not self.exists(path): raise errors.ResourceNotFound(path) @@ -828,6 +833,9 @@ def gettype(self, path): Returns: ~fs.enums.ResourceType: the type of the resource. + Raises: + fs.errors.ResourceNotFound: if ``path`` does not exist. + A type of a resource is an integer that identifies the what the resource references. The standard type integers may be one of the values in the `~fs.enums.ResourceType` enumerations. @@ -1201,8 +1209,8 @@ def opendir( ~fs.subfs.SubFS: A filesystem representing a sub-directory. Raises: - fs.errors.DirectoryExpected: If ``dst_path`` does not - exist or is not a directory. + fs.errors.ResourceNotFound: If ``path`` does not exist. + fs.errors.DirectoryExpected: If ``path`` is not a directory. """ from .subfs import SubFS @@ -1223,6 +1231,10 @@ def removetree(self, dir_path): Arguments: dir_path (str): Path to a directory on the filesystem. + Raises: + fs.errors.ResourceNotFound: If ``dir_path`` does not exist. + fs.errors.DirectoryExpected: If ``dir_path`` is not a directory. + Caution: A filesystem should never delete its root folder, so ``FS.removetree("/")`` has different semantics: the @@ -1497,11 +1509,10 @@ def validatepath(self, path): str: A normalized, absolute path. Raises: + fs.errors.InvalidPath: If the path is invalid. + fs.errors.FilesystemClosed: if the filesystem is closed. fs.errors.InvalidCharsInPath: If the path contains invalid characters. - fs.errors.InvalidPath: If the path is invalid. - fs.errors.FilesystemClosed: if the filesystem - is closed. """ self.check() @@ -1597,18 +1608,23 @@ def match(self, patterns, name): # type: (Optional[Iterable[Text]], Text) -> bool """Check if a name matches any of a list of wildcards. + If a filesystem is case *insensitive* (such as Windows) then + this method will perform a case insensitive match (i.e. ``*.py`` + will match the same names as ``*.PY``). Otherwise the match will + be case sensitive (``*.py`` and ``*.PY`` will match different + names). + Arguments: - patterns (list): A list of patterns, e.g. ``['*.py']`` + patterns (list, optional): A list of patterns, e.g. + ``['*.py']``, or `None` to match everything. name (str): A file or directory name (not a path) Returns: bool: `True` if ``name`` matches any of the patterns. - If a filesystem is case *insensitive* (such as Windows) then - this method will perform a case insensitive match (i.e. ``*.py`` - will match the same names as ``*.PY``). Otherwise the match will - be case sensitive (``*.py`` and ``*.PY`` will match different - names). + Raises: + TypeError: If ``patterns`` is a single string instead of + a list (or `None`). Example: >>> my_fs.match(['*.py'], '__init__.py') From 41fd7efe0c29f90df00c2f1b6c4d756e7e5f62cf Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 15:43:56 +0100 Subject: [PATCH 172/309] Illustrate how to use a proxy server in `FTPFS` --- fs/ftpfs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 35e39ca8..515709df 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -378,6 +378,12 @@ class FTPFS(FS): >>> ftp_fs = FTPFS("test.rebex.net", user="demo", passwd="password") >>> ftp_fs = fs.open_fs('ftp://demo:password@test.rebex.net') + Connecting via a proxy is supported. If using a FS URL, the proxy + URL will need to be added as a URL parameter:: + + >>> ftp_fs = FTPFS("ftp.ebi.ac.uk", proxy="test.rebex.net") + >>> ftp_fs = fs.open_fs('ftp://ftp.ebi.ac.uk/?proxy=test.rebex.net') + """ _meta = { From 41bbe3c29035efadb342fd0d4cc0e2a1077dfd2f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 16:01:40 +0100 Subject: [PATCH 173/309] Update `CHANGELOG.md` with changes introduced in #467 --- CHANGELOG.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7b98bc1..9bbf977f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). [#449](https://github.com/PyFilesystem/pyfilesystem2/pull/449). - `PathError` now supports wrapping an exception using the `exc` argument. Closes [#453](https://github.com/PyFilesystem/pyfilesystem2/issues/453). +- Better documentation of the `writable` parameter of `fs.open_fs`, and + hint about using `fs.wrap.read_only` when a read-only filesystem is + required. Closes [#441](https://github.com/PyFilesystem/pyfilesystem2/issues/441). ### Changed @@ -28,7 +31,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `FSTestCases` now builds the large data required for `upload` and `download` tests only once in order to reduce the total testing time. - `MemoryFS.move` and `MemoryFS.movedir` will now avoid copying data. - Closes [#452](https://github.com/PyFilesystem/pyfilesystem2/issues/452). + Closes [#452](https://github.com/PyFilesystem/pyfilesystem2/issues/452). +- `FS.removetree("/")` behaviour has been standardized in all filesystems, and + is expected to clear the contents of the root folder without deleting it. + Closes [#471](https://github.com/PyFilesystem/pyfilesystem2/issues/471). +- `FS.getbasic` is now deprecated, as it is redundant with `FS.getinfo`, + and `FS.getinfo` is now explicitly expected to return the *basic* info + namespace unconditionally. Closes [#469](https://github.com/PyFilesystem/pyfilesystem2/issues/469). ### Fixed @@ -40,8 +49,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - `WrapCachedDir.isdir` and `WrapCachedDir.isfile` raising a `ResourceNotFound` error on non-existing path ([#470](https://github.com/PyFilesystem/pyfilesystem2/pull/470)). - `FTPFS` not listing certain entries with sticky/SUID/SGID permissions set by Linux server ([#473](https://github.com/PyFilesystem/pyfilesystem2/pull/473)). Closes [#451](https://github.com/PyFilesystem/pyfilesystem2/issues/451). -- `scandir` iterator not being closed explicitly in `OSFS.scandir`, occasionally causing a `ResourceWarning` +- `scandir` iterator not being closed explicitly in `OSFS.scandir`, occasionally causing a `ResourceWarning` to be thrown. Closes [#311](https://github.com/PyFilesystem/pyfilesystem2/issues/311). +- Incomplete type annotations for the `temp_fs` parameter of `WriteTarFS` and `WriteZipFS`. + Closes [#410](https://github.com/PyFilesystem/pyfilesystem2/issues/410). ## [2.4.12] - 2021-01-14 From 736f42b682a366ccb2a0e7cf4f59386256614d9c Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 16:34:26 +0100 Subject: [PATCH 174/309] Add GitHub Actions workflow to publish releases to PyPI --- .github/workflows/package.yml | 122 ++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000..60b02ae9 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,122 @@ +name: Package + +on: + - push + - pull_request + +jobs: + + build-wheel: + runs-on: ubuntu-latest + name: Build wheel distribution + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Update build dependencies + run: python -m pip install -U pip wheel setuptools + - name: Build wheel distribution + run: python setup.py bdist_wheel + - name: Store built wheel + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/* + + build-sdist: + runs-on: ubuntu-latest + name: Build source distribution + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Update build dependencies + run: python -m pip install -U pip wheel setuptools + - name: Build source distribution + run: python setup.py sdist + - name: Store source distribution + uses: actions/upload-artifact@v2 + with: + name: dist + path: dist/* + + test-sdist: + runs-on: ubuntu-latest + name: Test source distribution + needs: + - build-sdist + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + - name: Download source distribution + uses: actions/download-artifact@v2 + with: + name: dist + path: dist + - name: Install source distribution + run: python -m pip install dist/fs-*.tar.gz + - name: Remove source code + run: rm -rvd fs + - name: Install test requirements + run: python -m pip install -r tests/requirements.txt + - name: Test installed package + run: python -m unittest discover -vv + + test-wheel: + runs-on: ubuntu-latest + name: Test wheel distribution + needs: + - build-wheel + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + submodules: true + - name: Download source distribution + uses: actions/download-artifact@v2 + with: + name: dist + path: dist + - name: Install source distribution + run: python -m pip install dist/fs-*.whl + - name: Remove source code + run: rm -rvd fs + - name: Install test requirements + run: python -m pip install -r tests/requirements.txt + - name: Test installed package + run: python -m unittest discover -vv + + upload: + environment: PyPI + runs-on: ubuntu-latest + name: Upload + needs: + - build-sdist + - build-wheel + - test-sdist + - test-wheel + steps: + - name: Download built distributions + uses: actions/download-artifact@v2 + with: + name: dist + path: dist + - name: Publish distributions to PyPI + if: startsWith(github.ref, 'refs/tags/v') + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + skip_existing: false From 7fd65582866ce0292c9a0b6d3bc44079ca598eaa Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 16:44:42 +0100 Subject: [PATCH 175/309] Make `package.yml` workflow test code on Python 3.9 --- .github/workflows/package.yml | 10 +++++++++- tests/mark.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 60b02ae9..0d1b6d04 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -60,6 +60,10 @@ jobs: uses: actions/checkout@v2 with: submodules: true + - name: Setup Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 - name: Download source distribution uses: actions/download-artifact@v2 with: @@ -84,7 +88,11 @@ jobs: uses: actions/checkout@v2 with: submodules: true - - name: Download source distribution + - name: Setup Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Download wheel distribution uses: actions/download-artifact@v2 with: name: dist diff --git a/tests/mark.py b/tests/mark.py index 4ac89d59..5bd8f12d 100644 --- a/tests/mark.py +++ b/tests/mark.py @@ -1,2 +1,2 @@ -def slow(self): - pass +def slow(cls): + return cls From 01f2d344696c421808fbe25dc27c1280f6398598 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 27 Mar 2021 17:37:32 +0100 Subject: [PATCH 176/309] Removed previous change from setup.cfg. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index f667b1a1..b90ebe47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -133,7 +133,6 @@ sitepackages = false skip_missing_interpreters = true requires = setuptools >=38.3.0 - parameterized ~=0.8 [testenv] commands = pytest --cov={toxinidir}/fs {posargs} {toxinidir}/tests From 0d0aa74383ba40823b783030d984908c87c4ec9e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 18:12:47 +0100 Subject: [PATCH 177/309] Make `package.yml` workflow only run on tagged commits --- .github/workflows/package.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 0d1b6d04..9afb05be 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -1,8 +1,9 @@ name: Package on: - - push - - pull_request + push: + tags: + - 'v2.*' jobs: @@ -97,7 +98,7 @@ jobs: with: name: dist path: dist - - name: Install source distribution + - name: Install wheel distribution run: python -m pip install dist/fs-*.whl - name: Remove source code run: rm -rvd fs From cc3c2807b1c207e651c6727c218fac960fa548fa Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 27 Mar 2021 18:17:57 +0100 Subject: [PATCH 178/309] Release v2.4.13 --- CHANGELOG.md | 3 +++ fs/_version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bbf977f..fcbbbc5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + +## [2.4.13] - 2021-03-27 + ### Added - Added FTP over TLS (FTPS) support to FTPFS. diff --git a/fs/_version.py b/fs/_version.py index 7dc85736..8f74ece1 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.12" +__version__ = "2.4.13" From b466bcb572311bc7b32b70f715ca5510a0e2c690 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 1 Apr 2021 12:26:01 +0200 Subject: [PATCH 179/309] Added `preserve_time` parameter that was missing from `MemoryFS.move` and `MemoryFS.movedir`. --- fs/memoryfs.py | 4 ++-- tests/test_osfs.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 7965a135..60c9b365 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -444,7 +444,7 @@ def makedir( parent_dir.set_entry(dir_name, new_dir) return self.opendir(path) - def move(self, src_path, dst_path, overwrite=False): + def move(self, src_path, dst_path, overwrite=False, preserve_time=False): src_dir, src_name = split(self.validatepath(src_path)) dst_dir, dst_name = split(self.validatepath(dst_path)) @@ -465,7 +465,7 @@ def move(self, src_path, dst_path, overwrite=False): dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) - def movedir(self, src_path, dst_path, create=False): + def movedir(self, src_path, dst_path, create=False, preserve_time=False): src_dir, src_name = split(self.validatepath(src_path)) dst_dir, dst_name = split(self.validatepath(dst_path)) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 704dee5e..ec5d8957 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -113,7 +113,6 @@ def test_copy_preserve_time(self): dst_info = self.fs.getinfo("bar/file.txt", namespaces) self.assertEqual(dst_info.modified, src_info.modified) - self.assertEqual(dst_info.accessed, src_info.accessed) self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) @unittest.skipUnless(osfs.sendfile, "sendfile not supported") From e6789572a2b7c5c1c56516c85c26050e3ff2358e Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 1 Apr 2021 12:40:41 +0200 Subject: [PATCH 180/309] For the newly added `preserve_time` parameter, reserved atime and ctime guarantees. As it turns out, there are platforms which do not allow changing either atime or ctime to custom values. Thus, the proposed behavior was impossible to implement. `preserve_time` now only makes guarantees about mtime. --- CHANGELOG.md | 3 +++ fs/base.py | 16 ++++++++-------- fs/copy.py | 32 ++++++++++++++++---------------- fs/mirror.py | 4 ++-- fs/move.py | 12 ++++++------ tests/test_copy.py | 18 +++--------------- tests/test_memoryfs.py | 6 ++---- tests/test_move.py | 13 ++----------- tests/test_osfs.py | 3 +-- tests/test_wrap.py | 8 ++++---- 10 files changed, 47 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bbf977f..41f3ff18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Better documentation of the `writable` parameter of `fs.open_fs`, and hint about using `fs.wrap.read_only` when a read-only filesystem is required. Closes [#441](https://github.com/PyFilesystem/pyfilesystem2/issues/441). +- Copy and move operations now provide a parameter `preserve_time` that, when + passed as `True`, makes sure the "mtime" of the destination file will be + the same as that of the source file. ### Changed diff --git a/fs/base.py b/fs/base.py index 7b2bf099..86644377 100644 --- a/fs/base.py +++ b/fs/base.py @@ -409,8 +409,8 @@ def copy( dst_path (str): Path to destination file. overwrite (bool): If `True`, overwrite the destination file if it exists (defaults to `False`). - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). Raises: fs.errors.DestinationExists: If ``dst_path`` exists, @@ -443,8 +443,8 @@ def copydir( dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created if it doesn't exist already (defaults to `False`). - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). Raises: fs.errors.ResourceNotFound: If the ``dst_path`` @@ -1054,8 +1054,8 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): dst_path (str): Path to destination directory. create (bool): If `True`, then ``dst_path`` will be created if it doesn't exist already (defaults to `False`). - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). Raises: fs.errors.ResourceNotFound: if ``dst_path`` does not exist, @@ -1122,8 +1122,8 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): file will be written to. overwrite (bool): If `True`, destination path will be overwritten if it exists. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). Raises: fs.errors.FileExpected: If ``src_path`` maps to a diff --git a/fs/copy.py b/fs/copy.py index 5a252891..600d301b 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -40,8 +40,8 @@ def copy_fs( dst_path)``. workers (int): Use `worker` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ return copy_dir( @@ -76,8 +76,8 @@ def copy_fs_if_newer( dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ return copy_dir_if_newer( @@ -138,8 +138,8 @@ def copy_file( src_path (str): Path to a file on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). """ with manage_fs(src_fs, writeable=False) as _src_fs: @@ -182,8 +182,8 @@ def copy_file_internal( src_path (str): Path to a file on the source filesystem. dst_fs (FS): Destination filesystem. dst_path (str): Path to a file on the destination filesystem. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). """ if src_fs is dst_fs: @@ -223,8 +223,8 @@ def copy_file_if_newer( src_path (str): Path to a file on the source filesystem. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on the destination filesystem. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). Returns: bool: `True` if the file copy was executed, `False` otherwise. @@ -273,8 +273,8 @@ def copy_structure( walker (~fs.walk.Walker, optional): A walker object that will be used to scan for files in ``src_fs``. Set this if you only want to consider a sub-set of the resources in ``src_fs``. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resource (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resource (defaults to `False`). """ walker = walker or Walker() @@ -311,8 +311,8 @@ def copy_dir( ``(src_fs, src_path, dst_fs, dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ on_copy = on_copy or (lambda *args: None) @@ -383,8 +383,8 @@ def copy_dir_if_newer( ``(src_fs, src_path, dst_fs, dst_path)``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ on_copy = on_copy or (lambda *args: None) diff --git a/fs/mirror.py b/fs/mirror.py index 1c674cbc..15b067fe 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -74,8 +74,8 @@ def mirror( workers (int): Number of worker threads used (0 for single threaded). Set to a relatively low number for network filesystems, 4 would be a good start. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ diff --git a/fs/move.py b/fs/move.py index 56dbcb98..56e9e5ca 100644 --- a/fs/move.py +++ b/fs/move.py @@ -29,8 +29,8 @@ def move_fs( dst_fs (FS or str): Destination filesystem (instance or URL). workers (int): Use `worker` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ move_dir(src_fs, "/", dst_fs, "/", workers=workers, preserve_time=preserve_time) @@ -51,8 +51,8 @@ def move_file( src_path (str): Path to a file on ``src_fs``. dst_fs (FS or str): Destination filesystem (instance or URL). dst_path (str): Path to a file on ``dst_fs``. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ with manage_fs(src_fs) as _src_fs: @@ -93,8 +93,8 @@ def move_dir( dst_path (str): Path to a directory on ``dst_fs``. workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. - preserve_time (bool): If `True`, try to preserve atime, ctime, - and mtime of the resources (defaults to `False`). + preserve_time (bool): If `True`, try to preserve mtime of the + resources (defaults to `False`). """ diff --git a/tests/test_copy.py b/tests/test_copy.py index f2f279ba..5c4fd0c8 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -17,7 +17,7 @@ class TestCopy(unittest.TestCase): @parameterized.expand([(0,), (1,), (2,), (4,)]) def test_copy_fs(self, workers): - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") @@ -37,13 +37,7 @@ def test_copy_fs(self, workers): dst_file1_info = dst_fs.getinfo("test.txt", namespaces) dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) self.assertEqual(dst_file1_info.modified, src_file1_info.modified) - self.assertEqual( - dst_file1_info.metadata_changed, src_file1_info.metadata_changed - ) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual( - dst_file2_info.metadata_changed, src_file2_info.metadata_changed - ) def test_copy_value_error(self): src_fs = open_fs("mem://") @@ -52,7 +46,7 @@ def test_copy_value_error(self): fs.copy.copy_fs(src_fs, dst_fs, workers=-1) def test_copy_dir0(self): - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") @@ -69,13 +63,10 @@ def test_copy_dir0(self): dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual( - dst_file2_info.metadata_changed, src_file2_info.metadata_changed - ) @parameterized.expand([(0,), (1,), (2,), (4,)]) def test_copy_dir(self, workers): - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") @@ -94,9 +85,6 @@ def test_copy_dir(self, workers): dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual( - dst_file2_info.metadata_changed, src_file2_info.metadata_changed - ) def test_copy_large(self): data1 = b"foo" * 512 * 1024 diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index ea6b62c9..bb5096f9 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -72,7 +72,7 @@ def test_copy_preserve_time(self): self.fs.makedir("bar") self.fs.touch("foo/file.txt") - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_info = self.fs.getinfo("foo/file.txt", namespaces) self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) @@ -80,11 +80,9 @@ def test_copy_preserve_time(self): dst_info = self.fs.getinfo("bar/file.txt", namespaces) self.assertEqual(dst_info.modified, src_info.modified) - self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) - -class TestMemoryFile(unittest.TestCase): +class TestMemoryFile(unittest.TestCase): def setUp(self): self.fs = memoryfs.MemoryFS() diff --git a/tests/test_move.py b/tests/test_move.py index 04a93628..5d5a4059 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -11,7 +11,7 @@ @parameterized_class(("preserve_time",), [(True,), (False,)]) class TestMove(unittest.TestCase): def test_move_fs(self): - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") @@ -32,16 +32,10 @@ def test_move_fs(self): dst_file1_info = dst_fs.getinfo("test.txt", namespaces) dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) self.assertEqual(dst_file1_info.modified, src_file1_info.modified) - self.assertEqual( - dst_file1_info.metadata_changed, src_file1_info.metadata_changed - ) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual( - dst_file2_info.metadata_changed, src_file2_info.metadata_changed - ) def test_move_dir(self): - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") @@ -60,6 +54,3 @@ def test_move_dir(self): if self.preserve_time: dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) - self.assertEqual( - dst_file2_info.metadata_changed, src_file2_info.metadata_changed - ) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index ec5d8957..6d2b11f6 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -105,7 +105,7 @@ def test_copy_preserve_time(self): raw_info = {"details": {"accessed": now, "modified": now}} self.fs.setinfo("foo/file.txt", raw_info) - namespaces = ("details", "accessed", "metadata_changed", "modified") + namespaces = ("details", "modified") src_info = self.fs.getinfo("foo/file.txt", namespaces) self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) @@ -113,7 +113,6 @@ def test_copy_preserve_time(self): dst_info = self.fs.getinfo("bar/file.txt", namespaces) self.assertEqual(dst_info.modified, src_info.modified) - self.assertEqual(dst_info.metadata_changed, src_info.metadata_changed) @unittest.skipUnless(osfs.sendfile, "sendfile not supported") @unittest.skipIf( diff --git a/tests/test_wrap.py b/tests/test_wrap.py index 89a91187..e11ded4a 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -177,7 +177,7 @@ def test_scandir(self): ] with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) scandir.assert_not_called() @@ -187,7 +187,7 @@ def test_isdir(self): self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) # is file self.assertFalse(self.cached.isdir("spam")) # doesn't exist - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) @@ -199,7 +199,7 @@ def test_isfile(self): self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) # is dir self.assertFalse(self.cached.isfile("spam")) # doesn't exist - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) @@ -211,7 +211,7 @@ def test_getinfo(self): self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) self.assertNotFound(self.cached.getinfo, "spam") - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) From 9b432a2612f8c0dc8292a88aefec185e1a017c77 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 1 Apr 2021 13:01:22 +0200 Subject: [PATCH 181/309] Fixed `OSFS.copy(..., preserve_time=True)`. --- fs/base.py | 6 +++--- fs/copy.py | 6 +++--- fs/osfs.py | 3 +++ tests/test_osfs.py | 7 +++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fs/base.py b/fs/base.py index 86644377..1363931a 100644 --- a/fs/base.py +++ b/fs/base.py @@ -22,7 +22,7 @@ import six from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard -from .copy import copy_cmtime +from .copy import copy_mtime from .glob import BoundGlobber from .mode import validate_open_mode from .path import abspath, join, normpath @@ -426,7 +426,7 @@ def copy( with closing(self.open(src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - copy_cmtime(self, src_path, self, dst_path) + copy_mtime(self, src_path, self, dst_path) def copydir( self, @@ -1155,7 +1155,7 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): with self.open(src_path, "rb") as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - copy_cmtime(self, src_path, self, dst_path) + copy_mtime(self, src_path, self, dst_path) self.remove(src_path) def open( diff --git a/fs/copy.py b/fs/copy.py index 600d301b..15a9fc84 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -159,7 +159,7 @@ def copy_file( else: with _src_fs.openbin(src_path) as read_file: _dst_fs.upload(dst_path, read_file) - copy_cmtime(_src_fs, src_path, _dst_fs, dst_path) + copy_mtime(_src_fs, src_path, _dst_fs, dst_path) def copy_file_internal( @@ -199,7 +199,7 @@ def copy_file_internal( with src_fs.openbin(src_path) as read_file: dst_fs.upload(dst_path, read_file) - copy_cmtime(src_fs, src_path, dst_fs, dst_path) + copy_mtime(src_fs, src_path, dst_fs, dst_path) def copy_file_if_newer( @@ -445,7 +445,7 @@ def dst(): on_copy(_src_fs, dir_path, _dst_fs, copy_path) -def copy_cmtime( +def copy_mtime( src_fs, # type: Union[FS, Text] src_path, # type: Text dst_fs, # type: Union[FS, Text] diff --git a/fs/osfs.py b/fs/osfs.py index 359876dc..4ffe7188 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -49,6 +49,7 @@ from .mode import Mode, validate_open_mode from .errors import FileExpected, NoURL from ._url_tools import url_quote +from .copy import copy_mtime if typing.TYPE_CHECKING: from typing import ( @@ -452,6 +453,8 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): while sent > 0: sent = sendfile(fd_dst, fd_src, offset, maxsize) offset += sent + if preserve_time: + copy_mtime(self, src_path, self, dst_path) except OSError as e: # the error is not a simple "sendfile not supported" error if e.errno not in self._sendfile_error_codes: diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 6d2b11f6..ff3af9e5 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -100,10 +100,9 @@ def test_expand_vars(self): def test_copy_preserve_time(self): self.fs.makedir("foo") self.fs.makedir("bar") - now = time.time() - 10000 - if not self.fs.create("foo/file.txt"): - raw_info = {"details": {"accessed": now, "modified": now}} - self.fs.setinfo("foo/file.txt", raw_info) + self.fs.create("foo/file.txt") + raw_info = {"details": {"modified": time.time() - 10000}} + self.fs.setinfo("foo/file.txt", raw_info) namespaces = ("details", "modified") src_info = self.fs.getinfo("foo/file.txt", namespaces) From 4a7591ea3e98acebc8a5f72673d1e7c9439df8a3 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 1 Apr 2021 13:21:14 +0200 Subject: [PATCH 182/309] Make AppVeyor install dependencies from `tests/requirements.txt` --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a8f3f337..78868972 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,10 +20,11 @@ environment: install: # We need wheel installed to build wheels - - "%PYTHON%\\python.exe -m pip install pytest pytest-randomly pytest-cov psutil pyftpdlib mock" + - "%PYTHON%\\python.exe -m pip install -U pip wheel setuptools" + - "%PYTHON%\\python.exe -m pip install -r tests/requirements.txt" - "%PYTHON%\\python.exe setup.py install" build: off test_script: - - "%PYTHON%\\python.exe -m pytest -v tests" + - "%PYTHON%\\python.exe -m unittest discover -vv" From f6a6195d10452da79b0068a7003042fa03c87ab4 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 1 Apr 2021 13:41:07 +0200 Subject: [PATCH 183/309] Revert back to testing with `pytest` in AppVeyor CI --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 78868972..e55b138a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -21,10 +21,11 @@ environment: install: # We need wheel installed to build wheels - "%PYTHON%\\python.exe -m pip install -U pip wheel setuptools" + - "%PYTHON%\\python.exe -m pip install pytest" - "%PYTHON%\\python.exe -m pip install -r tests/requirements.txt" - "%PYTHON%\\python.exe setup.py install" build: off test_script: - - "%PYTHON%\\python.exe -m unittest discover -vv" + - "%PYTHON%\\python.exe -m pytest" From 09cf4dc628e98df294835abf91813ea4e17008a1 Mon Sep 17 00:00:00 2001 From: atollk Date: Wed, 7 Apr 2021 19:28:13 +0200 Subject: [PATCH 184/309] Added support for MFMT command to `FTPFS.setinfo` to set the last modified time of a file. --- fs/copy.py | 2 +- fs/ftpfs.py | 27 +++++++++++++++++++++++---- tests/test_ftpfs.py | 26 +++++++++++++++++++++++--- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 15a9fc84..0260b30b 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -461,7 +461,7 @@ def copy_mtime( dst_path (str): Path to a directory on the destination filesystem. """ - namespaces = ("details", "metadata_changed", "modified") + namespaces = ("details",) with manage_fs(src_fs, writeable=False) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: src_meta = _src_fs.getinfo(src_path, namespaces) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 515709df..79462de0 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -6,6 +6,7 @@ import array import calendar +import datetime import io import itertools import socket @@ -19,8 +20,7 @@ from ftplib import FTP_TLS except ImportError as err: FTP_TLS = err # type: ignore -from ftplib import error_perm -from ftplib import error_temp +from ftplib import error_perm, error_temp from typing import cast from six import PY2 @@ -836,8 +836,27 @@ def writebytes(self, path, contents): def setinfo(self, path, info): # type: (Text, RawInfo) -> None - if not self.exists(path): - raise errors.ResourceNotFound(path) + current_info = self.getinfo(path) + if current_info.is_file and "MFMT" in self.features: + mtime = 0.0 + if "modified" in info: + mtime = float(cast(float, info["modified"]["modified"])) + if "details" in info: + mtime = float(cast(float, info["details"]["modified"])) + if mtime: + with ftp_errors(self, path): + cmd = ( + "MFMT " + + datetime.datetime.fromtimestamp(mtime).strftime( + "%Y%m%d%H%M%S" + ) + + " " + + _encode(path, self.ftp.encoding) + ) + try: + self.ftp.sendcmd(cmd) + except error_perm: + pass def readbytes(self, path): # type: (Text) -> bytes diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index b0e0c1f0..86d72ef0 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -11,6 +11,7 @@ import time import unittest import uuid +import datetime try: from unittest import mock @@ -144,7 +145,6 @@ def test_manager_with_host(self): @mark.slow @unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestFTPFS(FSTestCases, unittest.TestCase): - user = "user" pasw = "1234" @@ -162,7 +162,7 @@ def setUpClass(cls): cls.server.shutdown_after = -1 cls.server.handler.authorizer = DummyAuthorizer() cls.server.handler.authorizer.add_user( - cls.user, cls.pasw, cls._temp_path, perm="elradfmw" + cls.user, cls.pasw, cls._temp_path, perm="elradfmwT" ) cls.server.handler.authorizer.add_anonymous(cls._temp_path) cls.server.start() @@ -223,6 +223,27 @@ def test_geturl(self): ), ) + def test_setinfo(self): + # TODO: temporary test, since FSTestCases.test_setinfo is broken. + self.fs.create("bar") + time1 = time.mktime(datetime.datetime.now().timetuple()) + time2 = time1 + 60 + self.fs.setinfo("bar", {"details": {"modified": time1}}) + mtime1 = self.fs.getinfo("bar", ("details",)).modified + self.fs.setinfo("bar", {"details": {"modified": time2}}) + mtime2 = self.fs.getinfo("bar", ("details",)).modified + replacement = {} + if mtime1.microsecond == 0 or mtime2.microsecond == 0: + mtime1 = mtime1.replace(microsecond=0) + mtime2 = mtime2.replace(microsecond=0) + if mtime1.second == 0 or mtime2.second == 0: + mtime1 = mtime1.replace(second=0) + mtime2 = mtime2.replace(second=0) + mtime2_modified = mtime2.replace(minute=mtime2.minute - 1) + self.assertEqual( + mtime1.replace(**replacement), mtime2_modified.replace(**replacement) + ) + def test_host(self): self.assertEqual(self.fs.host, self.server.host) @@ -301,7 +322,6 @@ def test_features(self): @mark.slow @unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestAnonFTPFS(FSTestCases, unittest.TestCase): - user = "anonymous" pasw = "" From b4b3478e315806bfc387cd269052f80e5c4aaa10 Mon Sep 17 00:00:00 2001 From: atollk Date: Wed, 7 Apr 2021 19:59:28 +0200 Subject: [PATCH 185/309] Fixed the newly added preserve_time parameter in fs.copy actually being used, rather than being assumed to be True always. --- fs/copy.py | 15 ++++++++++----- tests/test_move.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 0260b30b..521517cf 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -45,7 +45,14 @@ def copy_fs( """ return copy_dir( - src_fs, "/", dst_fs, "/", walker=walker, on_copy=on_copy, workers=workers + src_fs, + "/", + dst_fs, + "/", + walker=walker, + on_copy=on_copy, + workers=workers, + preserve_time=preserve_time, ) @@ -199,7 +206,8 @@ def copy_file_internal( with src_fs.openbin(src_path) as read_file: dst_fs.upload(dst_path, read_file) - copy_mtime(src_fs, src_path, dst_fs, dst_path) + if preserve_time: + copy_mtime(src_fs, src_path, dst_fs, dst_path) def copy_file_if_newer( @@ -262,7 +270,6 @@ def copy_structure( src_fs, # type: Union[FS, Text] dst_fs, # type: Union[FS, Text] walker=None, # type: Optional[Walker] - preserve_time=False, # type: bool ): # type: (...) -> None """Copy directories (but not files) from ``src_fs`` to ``dst_fs``. @@ -273,8 +280,6 @@ def copy_structure( walker (~fs.walk.Walker, optional): A walker object that will be used to scan for files in ``src_fs``. Set this if you only want to consider a sub-set of the resources in ``src_fs``. - preserve_time (bool): If `True`, try to preserve mtime of the - resource (defaults to `False`). """ walker = walker or Walker() diff --git a/tests/test_move.py b/tests/test_move.py index 5d5a4059..72f48210 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -21,7 +21,7 @@ def test_move_fs(self): src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") - fs.move.move_fs(src_fs, dst_fs) + fs.move.move_fs(src_fs, dst_fs, preserve_time=self.preserve_time) self.assertTrue(dst_fs.isdir("foo/bar")) self.assertTrue(dst_fs.isfile("test.txt")) @@ -44,7 +44,7 @@ def test_move_dir(self): src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") - fs.move.move_dir(src_fs, "/foo", dst_fs, "/") + fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time) self.assertTrue(dst_fs.isdir("bar")) self.assertTrue(dst_fs.isfile("bar/baz.txt")) From e63cee23369fe7bca801ee5251bd45e1a9830c13 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 8 Apr 2021 03:21:17 +0200 Subject: [PATCH 186/309] Fix minor documentation issues in new `fs.copy` functions --- fs/copy.py | 67 ++++++++++++++---------------------------------------- 1 file changed, 17 insertions(+), 50 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 22c7a562..730a68a4 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -56,7 +56,8 @@ def copy_fs_if_newer( """Copy the contents of one filesystem to another, checking times. .. deprecated:: 2.5.0 - Use `~fs.copy_fs_if` with ``condition="newer"`` instead. + Use `~fs.copy.copy_fs_if` with ``condition="newer"`` instead. + """ warnings.warn( "copy_fs_if_newer is deprecated. Use copy_fs_if instead.", DeprecationWarning @@ -75,28 +76,6 @@ def copy_fs_if( # type: (...) -> None """Copy the contents of one filesystem to another, depending on a condition. - Depending on the value of ``strategy``, certain conditions must be fulfilled - for a file to be copied to ``dst_fs``. The following values - are supported: - - ``"always"`` - The source file is always copied. - ``"newer"`` - The last modification time of the source file must be newer than that - of the destination file. If either file has no modification time, the - copy is performed always. - ``"older"`` - The last modification time of the source file must be older than that - of the destination file. If either file has no modification time, the - copy is performed always. - ``"exists"`` - The source file is only copied if a file of the same path already - exists in ``dst_fs``. - ``"not_exists"`` - The source file is only copied if no file of the same path already - exists in ``dst_fs``. - - Arguments: src_fs (FS or str): Source filesystem (URL or instance). dst_fs (FS or str): Destination filesystem (URL or instance). @@ -110,6 +89,10 @@ def copy_fs_if( workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + See Also: + `~fs.copy.copy_file_if` for the full list of supported values for the + ``condition`` argument. + """ return copy_dir_if( src_fs, @@ -154,7 +137,8 @@ def copy_file_if_newer( """Copy a file from one filesystem to another, checking times. .. deprecated:: 2.5.0 - Use `~fs.copy_file_if` with ``condition="newer"`` instead. + Use `~fs.copy.copy_file_if` with ``condition="newer"`` instead. + """ warnings.warn( "copy_file_if_newer is deprecated. Use copy_file_if instead.", @@ -173,9 +157,9 @@ def copy_file_if( # type: (...) -> bool """Copy a file from one filesystem to another, depending on a condition. - Depending on the value of ``strategy``, certain conditions must be fulfilled - for a file to be copied to ``dst_fs``. The following values - are supported: + Depending on the value of ``condition``, certain requirements must + be fulfilled for a file to be copied to ``dst_fs``. The following + values are supported: ``"always"`` The source file is always copied. @@ -194,7 +178,6 @@ def copy_file_if( The source file is only copied if no file of the same path already exists in ``dst_fs``. - Arguments: src_fs (FS or str): Source filesystem (instance or URL). src_path (str): Path to a file on the source filesystem. @@ -335,7 +318,8 @@ def copy_dir_if_newer( """Copy a directory from one filesystem to another, checking times. .. deprecated:: 2.5.0 - Use `~fs.copy_dir_if` with ``condition="newer"`` instead. + Use `~fs.copy.copy_dir_if` with ``condition="newer"`` instead. + """ warnings.warn( "copy_dir_if_newer is deprecated. Use copy_dir_if instead.", DeprecationWarning @@ -356,27 +340,6 @@ def copy_dir_if( # type: (...) -> None """Copy a directory from one filesystem to another, depending on a condition. - Depending on the value of ``strategy``, certain conditions must be - fulfilled for a file to be copied to ``dst_fs``. The following values - are supported: - - ``"always"`` - The source file is always copied. - ``"newer"`` - The last modification time of the source file must be newer than that - of the destination file. If either file has no modification time, the - copy is performed always. - ``"older"`` - The last modification time of the source file must be older than that - of the destination file. If either file has no modification time, the - copy is performed always. - ``"exists"`` - The source file is only copied if a file of the same path already - exists in ``dst_fs``. - ``"not_exists"`` - The source file is only copied if no file of the same path already - exists in ``dst_fs``. - Arguments: src_fs (FS or str): Source filesystem (instance or URL). src_path (str): Path to a directory on the source filesystem. @@ -392,6 +355,10 @@ def copy_dir_if( workers (int): Use ``worker`` threads to copy data, or ``0`` (default) for a single-threaded copy. + See Also: + `~fs.copy.copy_file_if` for the full list of supported values for the + ``condition`` argument. + """ on_copy = on_copy or (lambda *args: None) walker = walker or Walker() From 2d04cca45328c178eae426b6bb4821020c907bf7 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Thu, 8 Apr 2021 03:26:42 +0200 Subject: [PATCH 187/309] Remove requests for `modified` namespace in `fs.copy._copy_is_necessary` --- fs/copy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 730a68a4..03108c00 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -332,7 +332,7 @@ def copy_dir_if( src_path, # type: Text dst_fs, # type: Union[FS, Text] dst_path, # type: Text - condition="always", # type: Text + condition, # type: Text walker=None, # type: Optional[Walker] on_copy=None, # type: Optional[_OnCopy] workers=0, # type: int @@ -398,7 +398,7 @@ def _copy_is_necessary( elif condition == "newer": try: - namespace = ("details", "modified") + namespace = ("details",) src_modified = src_fs.getinfo(src_path, namespace).modified dst_modified = dst_fs.getinfo(dst_path, namespace).modified except ResourceNotFound: @@ -412,7 +412,7 @@ def _copy_is_necessary( elif condition == "older": try: - namespace = ("details", "modified") + namespace = ("details",) src_modified = src_fs.getinfo(src_path, namespace).modified dst_modified = dst_fs.getinfo(dst_path, namespace).modified except ResourceNotFound: From 224952cf42f45470fcf3c3b1f57a1d3557552179 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 8 Apr 2021 11:44:01 +0200 Subject: [PATCH 188/309] Changes proposed by code review. --- fs/_bulk.py | 64 ++++++++++++++++++++++++++-------------------- fs/base.py | 10 +++++--- fs/copy.py | 21 +++++++-------- fs/memoryfs.py | 7 +++++ fs/mirror.py | 4 ++- fs/osfs.py | 4 +-- fs/wrapfs.py | 2 +- tests/test_move.py | 6 +++++ 8 files changed, 73 insertions(+), 45 deletions(-) diff --git a/fs/_bulk.py b/fs/_bulk.py index 7ee1a2c7..12eef85e 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -11,13 +11,14 @@ from six.moves.queue import Queue -from .copy import copy_file_internal +from .copy import copy_file_internal, copy_modified_time from .errors import BulkCopyFailed +from .tools import copy_file_data if typing.TYPE_CHECKING: from .base import FS from types import TracebackType - from typing import List, Optional, Text, Type + from typing import List, Optional, Text, Type, IO, Tuple class _Worker(threading.Thread): @@ -55,40 +56,32 @@ def __call__(self): class _CopyTask(_Task): """A callable that copies from one file another.""" - def __init__( - self, - src_fs, # type: FS - src_path, # type: Text - dst_fs, # type: FS - dst_path, # type: Text - preserve_time, # type: bool - ): - # type: (...) -> None - self.src_fs = src_fs - self.src_path = src_path - self.dst_fs = dst_fs - self.dst_path = dst_path - self.preserve_time = preserve_time + def __init__(self, src_file, dst_file): + # type: (IO, IO) -> None + self.src_file = src_file + self.dst_file = dst_file def __call__(self): # type: () -> None - copy_file_internal( - self.src_fs, - self.src_path, - self.dst_fs, - self.dst_path, - preserve_time=self.preserve_time, - ) + try: + copy_file_data(self.src_file, self.dst_file, chunk_size=1024 * 1024) + finally: + try: + self.src_file.close() + finally: + self.dst_file.close() class Copier(object): """Copy files in worker threads.""" - def __init__(self, num_workers=4): - # type: (int) -> None + def __init__(self, num_workers=4, preserve_time=False): + # type: (int, bool) -> None if num_workers < 0: raise ValueError("num_workers must be >= 0") self.num_workers = num_workers + self.preserve_time = preserve_time + self.all_tasks = [] # type: List[Tuple[FS, Text, FS, Text]] self.queue = None # type: Optional[Queue[_Task]] self.workers = [] # type: List[_Worker] self.errors = [] # type: List[Exception] @@ -97,7 +90,7 @@ def __init__(self, num_workers=4): def start(self): """Start the workers.""" if self.num_workers: - self.queue = Queue() + self.queue = Queue(maxsize=self.num_workers) self.workers = [_Worker(self) for _ in range(self.num_workers)] for worker in self.workers: worker.start() @@ -106,10 +99,18 @@ def start(self): def stop(self): """Stop the workers (will block until they are finished).""" if self.running and self.num_workers: + # Notify the workers that all tasks have arrived + # and wait for them to finish. for _worker in self.workers: self.queue.put(None) for worker in self.workers: worker.join() + + # If the "last modified" time is to be preserved, do it now. + if self.preserve_time: + for args in self.all_tasks: + copy_modified_time(*args) + # Free up references held by workers del self.workers[:] self.queue.join() @@ -139,8 +140,15 @@ def copy(self, src_fs, src_path, dst_fs, dst_path, preserve_time=False): if self.queue is None: # This should be the most performant for a single-thread copy_file_internal( - src_fs, src_path, dst_fs, dst_path, preserve_time=preserve_time + src_fs, src_path, dst_fs, dst_path, preserve_time=self.preserve_time ) else: - task = _CopyTask(src_fs, src_path, dst_fs, dst_path, preserve_time) + self.all_tasks.append((src_fs, src_path, dst_fs, dst_path)) + src_file = src_fs.openbin(src_path, "r") + try: + dst_file = dst_fs.openbin(dst_path, "w") + except Exception: + src_file.close() + raise + task = _CopyTask(src_file, dst_file) self.queue.put(task) diff --git a/fs/base.py b/fs/base.py index 1363931a..8d8ee4c8 100644 --- a/fs/base.py +++ b/fs/base.py @@ -22,7 +22,7 @@ import six from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard -from .copy import copy_mtime +from .copy import copy_modified_time from .glob import BoundGlobber from .mode import validate_open_mode from .path import abspath, join, normpath @@ -426,7 +426,8 @@ def copy( with closing(self.open(src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - copy_mtime(self, src_path, self, dst_path) + if preserve_time: + copy_modified_time(self, src_path, self, dst_path) def copydir( self, @@ -1150,12 +1151,15 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): except OSError: pass else: + if preserve_time: + copy_modified_time(self, src_path, self, dst_path) return with self._lock: with self.open(src_path, "rb") as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(dst_path, read_file) # type: ignore - copy_mtime(self, src_path, self, dst_path) + if preserve_time: + copy_modified_time(self, src_path, self, dst_path) self.remove(src_path) def open( diff --git a/fs/copy.py b/fs/copy.py index 521517cf..701d8215 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -166,7 +166,7 @@ def copy_file( else: with _src_fs.openbin(src_path) as read_file: _dst_fs.upload(dst_path, read_file) - copy_mtime(_src_fs, src_path, _dst_fs, dst_path) + copy_modified_time(_src_fs, src_path, _dst_fs, dst_path) def copy_file_internal( @@ -207,7 +207,7 @@ def copy_file_internal( dst_fs.upload(dst_path, read_file) if preserve_time: - copy_mtime(src_fs, src_path, dst_fs, dst_path) + copy_modified_time(src_fs, src_path, dst_fs, dst_path) def copy_file_if_newer( @@ -334,10 +334,11 @@ def dst(): from ._bulk import Copier with src() as _src_fs, dst() as _dst_fs: - _thread_safe = is_thread_safe(_src_fs, _dst_fs) - copier = Copier(num_workers=workers if _thread_safe else 0) - with copier: - with _src_fs.lock(), _dst_fs.lock(): + with _src_fs.lock(), _dst_fs.lock(): + _thread_safe = is_thread_safe(_src_fs, _dst_fs) + with Copier( + num_workers=workers if _thread_safe else 0, preserve_time=preserve_time + ) as copier: _dst_fs.makedir(_dst_path, recreate=True) for dir_path, dirs, files in walker.walk(_src_fs, _src_path): copy_path = combine(_dst_path, frombase(_src_path, dir_path)) @@ -351,7 +352,6 @@ def dst(): src_path, _dst_fs, dst_path, - preserve_time=preserve_time, ) on_copy(_src_fs, src_path, _dst_fs, dst_path) pass @@ -408,7 +408,9 @@ def dst(): with src() as _src_fs, dst() as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): _thread_safe = is_thread_safe(_src_fs, _dst_fs) - with Copier(num_workers=workers if _thread_safe else 0) as copier: + with Copier( + num_workers=workers if _thread_safe else 0, preserve_time=preserve_time + ) as copier: _dst_fs.makedir(_dst_path, recreate=True) namespace = ("details", "modified") dst_state = { @@ -445,12 +447,11 @@ def dst(): dir_path, _dst_fs, copy_path, - preserve_time=preserve_time, ) on_copy(_src_fs, dir_path, _dst_fs, copy_path) -def copy_mtime( +def copy_modified_time( src_fs, # type: Union[FS, Text] src_path, # type: Text dst_fs, # type: Union[FS, Text] diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 60c9b365..c34d60a7 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -15,6 +15,7 @@ from . import errors from .base import FS +from .copy import copy_modified_time from .enums import ResourceType, Seek from .info import Info from .mode import Mode @@ -465,6 +466,9 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) + if preserve_time: + copy_modified_time(self, src_path, self, dst_path) + def movedir(self, src_path, dst_path, create=False, preserve_time=False): src_dir, src_name = split(self.validatepath(src_path)) dst_dir, dst_name = split(self.validatepath(dst_path)) @@ -484,6 +488,9 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) + if preserve_time: + copy_modified_time(self, src_path, self, dst_path) + def openbin(self, path, mode="r", buffering=-1, **options): # type: (Text, Text, int, **Any) -> BinaryIO _mode = Mode(mode) diff --git a/fs/mirror.py b/fs/mirror.py index 15b067fe..dd00ff7b 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -87,7 +87,9 @@ def dst(): with src() as _src_fs, dst() as _dst_fs: _thread_safe = is_thread_safe(_src_fs, _dst_fs) - with Copier(num_workers=workers if _thread_safe else 0) as copier: + with Copier( + num_workers=workers if _thread_safe else 0, preserve_time=preserve_time + ) as copier: with _src_fs.lock(), _dst_fs.lock(): _mirror( _src_fs, diff --git a/fs/osfs.py b/fs/osfs.py index 4ffe7188..ac43471a 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -49,7 +49,7 @@ from .mode import Mode, validate_open_mode from .errors import FileExpected, NoURL from ._url_tools import url_quote -from .copy import copy_mtime +from .copy import copy_modified_time if typing.TYPE_CHECKING: from typing import ( @@ -454,7 +454,7 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): sent = sendfile(fd_dst, fd_src, offset, maxsize) offset += sent if preserve_time: - copy_mtime(self, src_path, self, dst_path) + copy_modified_time(self, src_path, self, dst_path) except OSError as e: # the error is not a simple "sendfile not supported" error if e.errno not in self._sendfile_error_codes: diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 57b605f3..00edd7af 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -186,7 +186,7 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): raise errors.ResourceNotFound(dst_path) if not src_fs.getinfo(_src_path).is_dir: raise errors.DirectoryExpected(src_path) - move_dir(src_fs, _src_path, dst_fs, _dst_path) + move_dir(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) def openbin(self, path, mode="r", buffering=-1, **options): # type: (Text, Text, int, **Any) -> BinaryIO diff --git a/tests/test_move.py b/tests/test_move.py index 72f48210..6b12b2b6 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -21,6 +21,9 @@ def test_move_fs(self): src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") + dst_fs.create("test.txt") + dst_fs.setinfo("test.txt", {"details": {"modified": 1000000}}) + fs.move.move_fs(src_fs, dst_fs, preserve_time=self.preserve_time) self.assertTrue(dst_fs.isdir("foo/bar")) @@ -44,6 +47,9 @@ def test_move_dir(self): src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) dst_fs = open_fs("mem://") + dst_fs.create("test.txt") + dst_fs.setinfo("test.txt", {"details": {"modified": 1000000}}) + fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time) self.assertTrue(dst_fs.isdir("bar")) From 8a1bf572f26759c5abecb5af5e540dee5fbf11b7 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 8 Apr 2021 11:56:16 +0200 Subject: [PATCH 189/309] Ran black to format code after merge. --- fs/copy.py | 50 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 9a9eb963..b9daea4b 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -45,7 +45,9 @@ def copy_fs( resources (defaults to `False`). """ - return copy_fs_if(src_fs, dst_fs, "always", walker, on_copy, workers, preserve_time=preserve_time) + return copy_fs_if( + src_fs, dst_fs, "always", walker, on_copy, workers, preserve_time=preserve_time + ) def copy_fs_if_newer( @@ -66,7 +68,9 @@ def copy_fs_if_newer( warnings.warn( "copy_fs_if_newer is deprecated. Use copy_fs_if instead.", DeprecationWarning ) - return copy_fs_if(src_fs, dst_fs, "newer", walker, on_copy, workers, preserve_time=preserve_time) + return copy_fs_if( + src_fs, dst_fs, "newer", walker, on_copy, workers, preserve_time=preserve_time + ) def copy_fs_if( @@ -135,7 +139,9 @@ def copy_file( resource (defaults to `False`). """ - copy_file_if(src_fs, src_path, dst_fs, dst_path, "always", preserve_time=preserve_time) + copy_file_if( + src_fs, src_path, dst_fs, dst_path, "always", preserve_time=preserve_time + ) def copy_file_if_newer( @@ -156,7 +162,9 @@ def copy_file_if_newer( "copy_file_if_newer is deprecated. Use copy_file_if instead.", DeprecationWarning, ) - return copy_file_if(src_fs, src_path, dst_fs, dst_path, "newer", preserve_time=preserve_time) + return copy_file_if( + src_fs, src_path, dst_fs, dst_path, "newer", preserve_time=preserve_time + ) def copy_file_if( @@ -210,7 +218,14 @@ def copy_file_if( _src_fs, src_path, _dst_fs, dst_path, condition ) if do_copy: - copy_file_internal(_src_fs, src_path, _dst_fs, dst_path, preserve_time=preserve_time, lock=True,) + copy_file_internal( + _src_fs, + src_path, + _dst_fs, + dst_path, + preserve_time=preserve_time, + lock=True, + ) return do_copy @@ -264,7 +279,6 @@ def _copy_locked(): _copy_locked() - def copy_structure( src_fs, # type: Union[FS, Text] dst_fs, # type: Union[FS, Text] @@ -327,7 +341,17 @@ def copy_dir( resources (defaults to `False`). """ - copy_dir_if(src_fs, src_path, dst_fs, dst_path, "always", walker, on_copy, workers, preserve_time=preserve_time) + copy_dir_if( + src_fs, + src_path, + dst_fs, + dst_path, + "always", + walker, + on_copy, + workers, + preserve_time=preserve_time, + ) def copy_dir_if_newer( @@ -350,7 +374,17 @@ def copy_dir_if_newer( warnings.warn( "copy_dir_if_newer is deprecated. Use copy_dir_if instead.", DeprecationWarning ) - copy_dir_if(src_fs, src_path, dst_fs, dst_path, "newer", walker, on_copy, workers, preserve_time=preserve_time) + copy_dir_if( + src_fs, + src_path, + dst_fs, + dst_path, + "newer", + walker, + on_copy, + workers, + preserve_time=preserve_time, + ) def copy_dir_if( From a20070840522116a99736539cb5c886e8ad3ca6a Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 8 Apr 2021 12:32:39 +0200 Subject: [PATCH 190/309] Fixed minor bug related to new `preserve_time` parameter in fs.copy. In a certain special case, the parameter was not passed on correctly but always `False` instead. --- fs/copy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/copy.py b/fs/copy.py index b9daea4b..95650b94 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -258,7 +258,7 @@ def copy_file_internal( if src_fs is dst_fs: # Same filesystem, so we can do a potentially optimized # copy - src_fs.copy(src_path, dst_path, overwrite=True) + src_fs.copy(src_path, dst_path, overwrite=True, preserve_time=preserve_time) return def _copy_locked(): From ebcd1a9d4bd9e41068d915014beb7a5ee0bbf119 Mon Sep 17 00:00:00 2001 From: atollk Date: Fri, 9 Apr 2021 17:30:32 +0200 Subject: [PATCH 191/309] Optimization in `FTPFS.setinfo`. Before, `setinfo` would always start by sending a MLST or LIST command to find whether the resources exists. This is actually not necessary if MFMT is sent anyway, so the command is now skipped in that case. --- fs/ftpfs.py | 45 +++++++++++++++++++++++++-------------------- tests/test_ftpfs.py | 31 ++++++++++++++----------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 79462de0..86f04958 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -836,27 +836,32 @@ def writebytes(self, path, contents): def setinfo(self, path, info): # type: (Text, RawInfo) -> None - current_info = self.getinfo(path) - if current_info.is_file and "MFMT" in self.features: - mtime = 0.0 + use_mfmt = False + if "MFMT" in self.features: + info_details = None if "modified" in info: - mtime = float(cast(float, info["modified"]["modified"])) - if "details" in info: - mtime = float(cast(float, info["details"]["modified"])) - if mtime: - with ftp_errors(self, path): - cmd = ( - "MFMT " - + datetime.datetime.fromtimestamp(mtime).strftime( - "%Y%m%d%H%M%S" - ) - + " " - + _encode(path, self.ftp.encoding) - ) - try: - self.ftp.sendcmd(cmd) - except error_perm: - pass + info_details = info["modified"] + elif "details" in info: + info_details = info["details"] + if info_details and "modified" in info_details: + use_mfmt = True + mtime = cast(float, info_details["modified"]) + + if use_mfmt: + with ftp_errors(self, path): + cmd = ( + "MFMT " + + datetime.datetime.utcfromtimestamp(mtime).strftime("%Y%m%d%H%M%S") + + " " + + _encode(path, self.ftp.encoding) + ) + try: + self.ftp.sendcmd(cmd) + except error_perm: + pass + else: + if not self.exists(path): + raise errors.ResourceNotFound(path) def readbytes(self, path): # type: (Text) -> bytes diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 86d72ef0..43028da7 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -3,6 +3,7 @@ from __future__ import print_function from __future__ import unicode_literals +import calendar import os import platform import shutil @@ -226,23 +227,19 @@ def test_geturl(self): def test_setinfo(self): # TODO: temporary test, since FSTestCases.test_setinfo is broken. self.fs.create("bar") - time1 = time.mktime(datetime.datetime.now().timetuple()) - time2 = time1 + 60 - self.fs.setinfo("bar", {"details": {"modified": time1}}) - mtime1 = self.fs.getinfo("bar", ("details",)).modified - self.fs.setinfo("bar", {"details": {"modified": time2}}) - mtime2 = self.fs.getinfo("bar", ("details",)).modified - replacement = {} - if mtime1.microsecond == 0 or mtime2.microsecond == 0: - mtime1 = mtime1.replace(microsecond=0) - mtime2 = mtime2.replace(microsecond=0) - if mtime1.second == 0 or mtime2.second == 0: - mtime1 = mtime1.replace(second=0) - mtime2 = mtime2.replace(second=0) - mtime2_modified = mtime2.replace(minute=mtime2.minute - 1) - self.assertEqual( - mtime1.replace(**replacement), mtime2_modified.replace(**replacement) - ) + original_modified = self.fs.getinfo("bar", ("details",)).modified + new_modified = original_modified - datetime.timedelta(hours=1) + new_modified_stamp = calendar.timegm(new_modified.timetuple()) + self.fs.setinfo("bar", {"details": {"modified": new_modified_stamp}}) + new_modified_get = self.fs.getinfo("bar", ("details",)).modified + if original_modified.microsecond == 0 or new_modified_get.microsecond == 0: + original_modified = original_modified.replace(microsecond=0) + new_modified_get = new_modified_get.replace(microsecond=0) + if original_modified.second == 0 or new_modified_get.second == 0: + original_modified = original_modified.replace(second=0) + new_modified_get = new_modified_get.replace(second=0) + new_modified_get = new_modified_get + datetime.timedelta(hours=1) + self.assertEqual(original_modified, new_modified_get) def test_host(self): self.assertEqual(self.fs.host, self.server.host) From b42d5ef0df26ef3f55f4832c953e965d85844e26 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 15 Apr 2021 09:28:19 +0200 Subject: [PATCH 192/309] Added support for "last modified time" in `getinfo`. A new namespace "modified" was introduced to the `getinfo` function to potentially use cheaper commands compared to the general "details" namespace. In particular, `FTPFS` now uses the MDTM command if supported by the server. --- CHANGELOG.md | 6 ++++++ fs/base.py | 21 +++++++++++++++++++++ fs/copy.py | 10 ++++------ fs/ftpfs.py | 14 ++++++++++++++ fs/info.py | 9 ++++++--- fs/osfs.py | 1 + tests/test_ftpfs.py | 19 +++++++++++++++++-- tests/test_memoryfs.py | 1 - tests/test_opener.py | 16 ++++++++++++++-- tests/test_wrap.py | 8 ++++---- 10 files changed, 87 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f933fb1..e210d618 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458). +### Changed + +- FTP servers that do not support the MLST command now try to use the MDTM command to + retrieve the last modification timestamp of a resource. + Closes [#456](https://github.com/PyFilesystem/pyfilesystem2/pull/456). + ### Fixed - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future. diff --git a/fs/base.py b/fs/base.py index 4966d0ca..0e4d8fed 100644 --- a/fs/base.py +++ b/fs/base.py @@ -678,6 +678,25 @@ def readtext( gettext = _new_name(readtext, "gettext") + def getmodified(self, path): + # type: (Text) -> Optional[datetime] + """Get the timestamp of the last modifying access of a resource. + + Arguments: + path (str): A path to a resource. + + Returns: + datetime: The timestamp of the last modification. + + The *modified timestamp* of a file is the point in time + that the file was last changed. Depending on the file system, + it might only have limited accuracy. + + """ + if self.getmeta().get("supports_mtime", False): + return self.getinfo(path, namespaces=["modified"]).modified + return self.getinfo(path, namespaces=["details"]).modified + def getmeta(self, namespace="standard"): # type: (Text) -> Mapping[Text, object] """Get meta information regarding a filesystem. @@ -715,6 +734,8 @@ def getmeta(self, namespace="standard"): read_only `True` if this filesystem is read only. supports_rename `True` if this filesystem supports an `os.rename` operation. + supports_mtime `True` if this filesystem supports a native + operation to retreive the "last modified" time. =================== ============================================ Most builtin filesystems will provide all these keys, and third- diff --git a/fs/copy.py b/fs/copy.py index 03108c00..9a6b9b62 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -398,9 +398,8 @@ def _copy_is_necessary( elif condition == "newer": try: - namespace = ("details",) - src_modified = src_fs.getinfo(src_path, namespace).modified - dst_modified = dst_fs.getinfo(dst_path, namespace).modified + src_modified = src_fs.getmodified(src_path) + dst_modified = dst_fs.getmodified(dst_path) except ResourceNotFound: return True else: @@ -412,9 +411,8 @@ def _copy_is_necessary( elif condition == "older": try: - namespace = ("details",) - src_modified = src_fs.getinfo(src_path, namespace).modified - dst_modified = dst_fs.getinfo(dst_path, namespace).modified + src_modified = src_fs.getmodified(src_path) + dst_modified = dst_fs.getmodified(dst_path) except ResourceNotFound: return True else: diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 515709df..459749c4 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -15,6 +15,7 @@ from contextlib import contextmanager from ftplib import FTP + try: from ftplib import FTP_TLS except ImportError as err: @@ -667,6 +668,18 @@ def getinfo(self, path, namespaces=None): } ) + if "modified" in namespaces: + if "basic" in namespaces or "details" in namespaces: + raise ValueError( + 'Cannot use the "modified" namespace in combination with others.' + ) + with self._lock: + with ftp_errors(self, path=path): + cmd = "MDTM " + _encode(self.validatepath(path), self.ftp.encoding) + response = self.ftp.sendcmd(cmd) + modified_info = {"modified": self._parse_ftp_time(response.split()[1])} + return Info({"modified": modified_info}) + if self.supports_mlst: with self._lock: with ftp_errors(self, path=path): @@ -692,6 +705,7 @@ def getmeta(self, namespace="standard"): if namespace == "standard": _meta = self._meta.copy() _meta["unicode_paths"] = "UTF8" in self.features + _meta["supports_mtime"] = "MDTM" in self.features return _meta def listdir(self, path): diff --git a/fs/info.py b/fs/info.py index 03bf27cd..6c6cdb0e 100644 --- a/fs/info.py +++ b/fs/info.py @@ -317,9 +317,12 @@ def modified(self): namespace is not in the Info. """ - self._require_namespace("details") - _time = self._make_datetime(self.get("details", "modified")) - return _time + try: + self._require_namespace("details") + return self._make_datetime(self.get("details", "modified")) + except MissingInfoNamespace: + self._require_namespace("modified") + return self._make_datetime(self.get("modified", "modified")) @property def created(self): diff --git a/fs/osfs.py b/fs/osfs.py index 5beb16bf..d5883948 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -144,6 +144,7 @@ def __init__( "network": False, "read_only": False, "supports_rename": True, + "supports_mtime": False, "thread_safe": True, "unicode_paths": os.path.supports_unicode_filenames, "virtual": False, diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index b0e0c1f0..37bb340b 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -144,7 +144,6 @@ def test_manager_with_host(self): @mark.slow @unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestFTPFS(FSTestCases, unittest.TestCase): - user = "user" pasw = "1234" @@ -243,6 +242,23 @@ def test_getmeta_unicode_path(self): del self.fs.features["UTF8"] self.assertFalse(self.fs.getmeta().get("unicode_paths")) + def test_getinfo_modified(self): + self.assertIn("MDTM", self.fs.features) + self.fs.create("bar") + mtime_detail = self.fs.getinfo("bar", ("basic", "details")).modified + mtime_modified = self.fs.getinfo("bar", ("modified",)).modified + # Microsecond and seconds might not actually be supported by all + # FTP commands, so we strip them before comparing if it looks + # like at least one of the two values does not contain them. + replacement = {} + if mtime_detail.microsecond == 0 or mtime_modified.microsecond == 0: + replacement["microsecond"] = 0 + if mtime_detail.second == 0 or mtime_modified.second == 0: + replacement["second"] = 0 + self.assertEqual( + mtime_detail.replace(**replacement), mtime_modified.replace(**replacement) + ) + def test_opener_path(self): self.fs.makedir("foo") self.fs.writetext("foo/bar", "baz") @@ -301,7 +317,6 @@ def test_features(self): @mark.slow @unittest.skipIf(platform.python_implementation() == "PyPy", "ftp unreliable with PyPy") class TestAnonFTPFS(FSTestCases, unittest.TestCase): - user = "anonymous" pasw = "" diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 31909f0f..980b694d 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -69,7 +69,6 @@ def test_close_mem_free(self): class TestMemoryFile(unittest.TestCase): - def setUp(self): self.fs = memoryfs.MemoryFS() diff --git a/tests/test_opener.py b/tests/test_opener.py index e7fae983..bc2f5cd7 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -300,14 +300,26 @@ def test_user_data_opener(self, app_dir): def test_open_ftp(self, mock_FTPFS): open_fs("ftp://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( - "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=False + "ftp.example.org", + passwd="bar", + port=21, + user="foo", + proxy=None, + timeout=10, + tls=False, ) @mock.patch("fs.ftpfs.FTPFS") def test_open_ftps(self, mock_FTPFS): open_fs("ftps://foo:bar@ftp.example.org") mock_FTPFS.assert_called_once_with( - "ftp.example.org", passwd="bar", port=21, user="foo", proxy=None, timeout=10, tls=True + "ftp.example.org", + passwd="bar", + port=21, + user="foo", + proxy=None, + timeout=10, + tls=True, ) @mock.patch("fs.ftpfs.FTPFS") diff --git a/tests/test_wrap.py b/tests/test_wrap.py index 89a91187..e11ded4a 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -177,7 +177,7 @@ def test_scandir(self): ] with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(sorted(self.cached.scandir("/"), key=key), expected) scandir.assert_not_called() @@ -187,7 +187,7 @@ def test_isdir(self): self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) # is file self.assertFalse(self.cached.isdir("spam")) # doesn't exist - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isdir("foo")) self.assertFalse(self.cached.isdir("egg")) @@ -199,7 +199,7 @@ def test_isfile(self): self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) # is dir self.assertFalse(self.cached.isfile("spam")) # doesn't exist - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertTrue(self.cached.isfile("egg")) self.assertFalse(self.cached.isfile("foo")) @@ -211,7 +211,7 @@ def test_getinfo(self): self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) self.assertNotFound(self.cached.getinfo, "spam") - scandir.assert_has_calls([mock.call('/', namespaces=None, page=None)]) + scandir.assert_has_calls([mock.call("/", namespaces=None, page=None)]) with mock.patch.object(self.fs, "scandir", wraps=self.fs.scandir) as scandir: self.assertEqual(self.cached.getinfo("foo"), self.fs.getinfo("foo")) self.assertEqual(self.cached.getinfo("/"), self.fs.getinfo("/")) From 29354ef4cf601d9b8d89fbaef84c4f79b4d426e2 Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 15 Apr 2021 09:28:19 +0200 Subject: [PATCH 193/309] Added support for "last modified time" in `getinfo`. A new namespace "modified" was introduced to the `getinfo` function to potentially use cheaper commands compared to the general "details" namespace. In particular, `FTPFS` now uses the MDTM command if supported by the server. --- CHANGELOG.md | 6 ++++++ fs/base.py | 21 +++++++++++++++++++++ fs/copy.py | 10 ++++------ fs/ftpfs.py | 14 ++++++++++++++ fs/info.py | 9 ++++++--- fs/osfs.py | 1 + tests/test_ftpfs.py | 17 +++++++++++++++++ tests/test_memoryfs.py | 7 +++---- 8 files changed, 72 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24b0e66..67562281 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458). +### Changed + +- FTP servers that do not support the MLST command now try to use the MDTM command to + retrieve the last modification timestamp of a resource. + Closes [#456](https://github.com/PyFilesystem/pyfilesystem2/pull/456). + ### Fixed - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future. diff --git a/fs/base.py b/fs/base.py index 8d8ee4c8..6283b70f 100644 --- a/fs/base.py +++ b/fs/base.py @@ -697,6 +697,25 @@ def readtext( gettext = _new_name(readtext, "gettext") + def getmodified(self, path): + # type: (Text) -> Optional[datetime] + """Get the timestamp of the last modifying access of a resource. + + Arguments: + path (str): A path to a resource. + + Returns: + datetime: The timestamp of the last modification. + + The *modified timestamp* of a file is the point in time + that the file was last changed. Depending on the file system, + it might only have limited accuracy. + + """ + if self.getmeta().get("supports_mtime", False): + return self.getinfo(path, namespaces=["modified"]).modified + return self.getinfo(path, namespaces=["details"]).modified + def getmeta(self, namespace="standard"): # type: (Text) -> Mapping[Text, object] """Get meta information regarding a filesystem. @@ -734,6 +753,8 @@ def getmeta(self, namespace="standard"): read_only `True` if this filesystem is read only. supports_rename `True` if this filesystem supports an `os.rename` operation. + supports_mtime `True` if this filesystem supports a native + operation to retreive the "last modified" time. =================== ============================================ Most builtin filesystems will provide all these keys, and third- diff --git a/fs/copy.py b/fs/copy.py index 95650b94..6ffd83d7 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -463,9 +463,8 @@ def _copy_is_necessary( elif condition == "newer": try: - namespace = ("details",) - src_modified = src_fs.getinfo(src_path, namespace).modified - dst_modified = dst_fs.getinfo(dst_path, namespace).modified + src_modified = src_fs.getmodified(src_path) + dst_modified = dst_fs.getmodified(dst_path) except ResourceNotFound: return True else: @@ -477,9 +476,8 @@ def _copy_is_necessary( elif condition == "older": try: - namespace = ("details",) - src_modified = src_fs.getinfo(src_path, namespace).modified - dst_modified = dst_fs.getinfo(dst_path, namespace).modified + src_modified = src_fs.getmodified(src_path) + dst_modified = dst_fs.getmodified(dst_path) except ResourceNotFound: return True else: diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 86f04958..418d554a 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -16,6 +16,7 @@ from contextlib import contextmanager from ftplib import FTP + try: from ftplib import FTP_TLS except ImportError as err: @@ -667,6 +668,18 @@ def getinfo(self, path, namespaces=None): } ) + if "modified" in namespaces: + if "basic" in namespaces or "details" in namespaces: + raise ValueError( + 'Cannot use the "modified" namespace in combination with others.' + ) + with self._lock: + with ftp_errors(self, path=path): + cmd = "MDTM " + _encode(self.validatepath(path), self.ftp.encoding) + response = self.ftp.sendcmd(cmd) + modified_info = {"modified": self._parse_ftp_time(response.split()[1])} + return Info({"modified": modified_info}) + if self.supports_mlst: with self._lock: with ftp_errors(self, path=path): @@ -692,6 +705,7 @@ def getmeta(self, namespace="standard"): if namespace == "standard": _meta = self._meta.copy() _meta["unicode_paths"] = "UTF8" in self.features + _meta["supports_mtime"] = "MDTM" in self.features return _meta def listdir(self, path): diff --git a/fs/info.py b/fs/info.py index 03bf27cd..6c6cdb0e 100644 --- a/fs/info.py +++ b/fs/info.py @@ -317,9 +317,12 @@ def modified(self): namespace is not in the Info. """ - self._require_namespace("details") - _time = self._make_datetime(self.get("details", "modified")) - return _time + try: + self._require_namespace("details") + return self._make_datetime(self.get("details", "modified")) + except MissingInfoNamespace: + self._require_namespace("modified") + return self._make_datetime(self.get("modified", "modified")) @property def created(self): diff --git a/fs/osfs.py b/fs/osfs.py index ac43471a..c0de12cd 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -145,6 +145,7 @@ def __init__( "network": False, "read_only": False, "supports_rename": True, + "supports_mtime": False, "thread_safe": True, "unicode_paths": os.path.supports_unicode_filenames, "virtual": False, diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 43028da7..0501589e 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -261,6 +261,23 @@ def test_getmeta_unicode_path(self): del self.fs.features["UTF8"] self.assertFalse(self.fs.getmeta().get("unicode_paths")) + def test_getinfo_modified(self): + self.assertIn("MDTM", self.fs.features) + self.fs.create("bar") + mtime_detail = self.fs.getinfo("bar", ("basic", "details")).modified + mtime_modified = self.fs.getinfo("bar", ("modified",)).modified + # Microsecond and seconds might not actually be supported by all + # FTP commands, so we strip them before comparing if it looks + # like at least one of the two values does not contain them. + replacement = {} + if mtime_detail.microsecond == 0 or mtime_modified.microsecond == 0: + replacement["microsecond"] = 0 + if mtime_detail.second == 0 or mtime_modified.second == 0: + replacement["second"] = 0 + self.assertEqual( + mtime_detail.replace(**replacement), mtime_modified.replace(**replacement) + ) + def test_opener_path(self): self.fs.makedir("foo") self.fs.writetext("foo/bar", "baz") diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index bb5096f9..74cdc2d6 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -72,14 +72,13 @@ def test_copy_preserve_time(self): self.fs.makedir("bar") self.fs.touch("foo/file.txt") - namespaces = ("details", "modified") - src_info = self.fs.getinfo("foo/file.txt", namespaces) + src_info = self.fs.getmodified("foo/file.txt") self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) self.assertTrue(self.fs.exists("bar/file.txt")) - dst_info = self.fs.getinfo("bar/file.txt", namespaces) - self.assertEqual(dst_info.modified, src_info.modified) + dst_info = self.fs.getmodified("bar/file.txt") + self.assertEqual(dst_info, src_info) class TestMemoryFile(unittest.TestCase): From 3065813714dc8c2d3221538a5e7218e84e6bda37 Mon Sep 17 00:00:00 2001 From: atollk Date: Tue, 27 Apr 2021 09:41:07 +0200 Subject: [PATCH 194/309] Fixes from code review. --- CHANGELOG.md | 1 + docs/source/interface.rst | 1 + fs/base.py | 2 +- fs/ftpfs.py | 26 +++++++++++++++++--------- tests/test_memoryfs.py | 6 +++--- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67562281..968dc677 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458). +- Added `fs.base.FS.getmodified`. ### Changed diff --git a/docs/source/interface.rst b/docs/source/interface.rst index e2da135b..0d67c0c6 100644 --- a/docs/source/interface.rst +++ b/docs/source/interface.rst @@ -20,6 +20,7 @@ The following is a complete list of methods on PyFilesystem objects. * :meth:`~fs.base.FS.getdetails` Get details info namespace for a resource. * :meth:`~fs.base.FS.getinfo` Get info regarding a file or directory. * :meth:`~fs.base.FS.getmeta` Get meta information for a resource. +* :meth:`~fs.base.FS.getmodified` Get info regarding the last modified time of a resource. * :meth:`~fs.base.FS.getospath` Get path with encoding expected by the OS. * :meth:`~fs.base.FS.getsize` Get the size of a file. * :meth:`~fs.base.FS.getsyspath` Get the system path of a resource, if one exists. diff --git a/fs/base.py b/fs/base.py index 6283b70f..55edf3a4 100644 --- a/fs/base.py +++ b/fs/base.py @@ -754,7 +754,7 @@ def getmeta(self, namespace="standard"): supports_rename `True` if this filesystem supports an `os.rename` operation. supports_mtime `True` if this filesystem supports a native - operation to retreive the "last modified" time. + operation to retrieve the "last modified" time. =================== ============================================ Most builtin filesystems will provide all these keys, and third- diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 418d554a..260592ce 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -12,6 +12,7 @@ import socket import threading import typing +import warnings from collections import OrderedDict from contextlib import contextmanager from ftplib import FTP @@ -669,16 +670,23 @@ def getinfo(self, path, namespaces=None): ) if "modified" in namespaces: - if "basic" in namespaces or "details" in namespaces: - raise ValueError( - 'Cannot use the "modified" namespace in combination with others.' + if "details" in namespaces: + warnings.warn( + "FTPFS.getinfo called with both 'modified' and 'details'" + " namespace. The former will be ignored.", + UserWarning, ) - with self._lock: - with ftp_errors(self, path=path): - cmd = "MDTM " + _encode(self.validatepath(path), self.ftp.encoding) - response = self.ftp.sendcmd(cmd) - modified_info = {"modified": self._parse_ftp_time(response.split()[1])} - return Info({"modified": modified_info}) + else: + with self._lock: + with ftp_errors(self, path=path): + cmd = "MDTM " + _encode( + self.validatepath(path), self.ftp.encoding + ) + response = self.ftp.sendcmd(cmd) + modified_info = { + "modified": self._parse_ftp_time(response.split()[1]) + } + return Info({"modified": modified_info}) if self.supports_mlst: with self._lock: diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 74cdc2d6..67d92ac1 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -72,13 +72,13 @@ def test_copy_preserve_time(self): self.fs.makedir("bar") self.fs.touch("foo/file.txt") - src_info = self.fs.getmodified("foo/file.txt") + src_datetime = self.fs.getmodified("foo/file.txt") self.fs.copy("foo/file.txt", "bar/file.txt", preserve_time=True) self.assertTrue(self.fs.exists("bar/file.txt")) - dst_info = self.fs.getmodified("bar/file.txt") - self.assertEqual(dst_info, src_info) + dst_datetime = self.fs.getmodified("bar/file.txt") + self.assertEqual(dst_datetime, src_datetime) class TestMemoryFile(unittest.TestCase): From 98fd06eac359a20cc1a00d68a4681e38754f7a2b Mon Sep 17 00:00:00 2001 From: atollk Date: Tue, 27 Apr 2021 13:14:22 +0200 Subject: [PATCH 195/309] Fixes issue #477. Unit tests for `FS.setinfo` now function correctly, whereas they basically did nothing before. Also, a bug in OSFS was fixed which caused timestamps to lose precision with `setinfo`. --- fs/info.py | 3 +++ fs/osfs.py | 8 ++++---- fs/test.py | 26 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/fs/info.py b/fs/info.py index 03bf27cd..a2208cc5 100644 --- a/fs/info.py +++ b/fs/info.py @@ -135,6 +135,9 @@ def is_writeable(self, namespace, key): When creating an `Info` object, you can add a ``_write`` key to each raw namespace that lists which keys are writable or not. + In general, this means they are compatible with the `setinfo` + function of filesystem objects. + Arguments: namespace (str): A namespace identifier. key (str): A key within the namespace. diff --git a/fs/osfs.py b/fs/osfs.py index ac43471a..3e6292ef 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -665,10 +665,10 @@ def setinfo(self, path, info): if "details" in info: details = info["details"] if "accessed" in details or "modified" in details: - _accessed = typing.cast(int, details.get("accessed")) - _modified = typing.cast(int, details.get("modified", _accessed)) - accessed = int(_modified if _accessed is None else _accessed) - modified = int(_modified) + _accessed = typing.cast(float, details.get("accessed")) + _modified = typing.cast(float, details.get("modified", _accessed)) + accessed = float(_modified if _accessed is None else _accessed) + modified = float(_modified) if accessed is not None or modified is not None: with convert_os_errors("setinfo", path): os.utime(sys_path, (accessed, modified)) diff --git a/fs/test.py b/fs/test.py index e40e43cf..3e08f4ff 100644 --- a/fs/test.py +++ b/fs/test.py @@ -12,7 +12,6 @@ import io import itertools import json -import math import os import time import unittest @@ -1171,15 +1170,21 @@ def test_removetree_root(self): def test_setinfo(self): self.fs.create("birthday.txt") - now = math.floor(time.time()) + now = time.time() change_info = {"details": {"accessed": now + 60, "modified": now + 60 * 60}} self.fs.setinfo("birthday.txt", change_info) - new_info = self.fs.getinfo("birthday.txt", namespaces=["details"]).raw - if "accessed" in new_info.get("_write", []): - self.assertEqual(new_info["details"]["accessed"], now + 60) - if "modified" in new_info.get("_write", []): - self.assertEqual(new_info["details"]["modified"], now + 60 * 60) + new_info = self.fs.getinfo("birthday.txt", namespaces=["details"]) + can_write_acccess = new_info.is_writeable("details", "accessed") + can_write_modified = new_info.is_writeable("details", "modified") + if can_write_acccess: + self.assertAlmostEqual( + new_info.get("details", "accessed"), now + 60, places=4 + ) + if can_write_modified: + self.assertAlmostEqual( + new_info.get("details", "modified"), now + 60 * 60, places=4 + ) with self.assertRaises(errors.ResourceNotFound): self.fs.setinfo("nothing", {}) @@ -1188,10 +1193,11 @@ def test_settimes(self): self.fs.create("birthday.txt") self.fs.settimes("birthday.txt", accessed=datetime(2016, 7, 5)) info = self.fs.getinfo("birthday.txt", namespaces=["details"]) - writeable = info.get("details", "_write", []) - if "accessed" in writeable: + can_write_acccess = info.is_writeable("details", "accessed") + can_write_modified = info.is_writeable("details", "modified") + if can_write_acccess: self.assertEqual(info.accessed, datetime(2016, 7, 5, tzinfo=pytz.UTC)) - if "modified" in writeable: + if can_write_modified: self.assertEqual(info.modified, datetime(2016, 7, 5, tzinfo=pytz.UTC)) def test_touch(self): From 234851843b4a2f532f5696cffc1d7df04e58da63 Mon Sep 17 00:00:00 2001 From: atollk Date: Tue, 27 Apr 2021 13:15:46 +0200 Subject: [PATCH 196/309] Updated CHANGELOG.md. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24b0e66..f9abcd53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future. +- Fixed precision bug for timestamps in `fs.OSFS.setinfo`. ## [2.4.13] - 2021-03-27 From 762ee067c71b4bacf89e5056684775aaa26945fe Mon Sep 17 00:00:00 2001 From: atollk Date: Tue, 27 Apr 2021 14:11:48 +0200 Subject: [PATCH 197/309] Fixed py27 test. --- tests/test_osfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index ff3af9e5..c296b4d5 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -111,7 +111,7 @@ def test_copy_preserve_time(self): self.assertTrue(self.fs.exists("bar/file.txt")) dst_info = self.fs.getinfo("bar/file.txt", namespaces) - self.assertEqual(dst_info.modified, src_info.modified) + self.assertAlmostEqual(dst_info.modified, src_info.modified, places=2) @unittest.skipUnless(osfs.sendfile, "sendfile not supported") @unittest.skipIf( From b1511d270d099921f0eaa45b8ce65fc787c4797f Mon Sep 17 00:00:00 2001 From: atollk Date: Tue, 27 Apr 2021 14:34:35 +0200 Subject: [PATCH 198/309] Fixed py27-scandir test ? --- tests/test_osfs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_osfs.py b/tests/test_osfs.py index c296b4d5..7fdc5590 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -111,7 +111,8 @@ def test_copy_preserve_time(self): self.assertTrue(self.fs.exists("bar/file.txt")) dst_info = self.fs.getinfo("bar/file.txt", namespaces) - self.assertAlmostEqual(dst_info.modified, src_info.modified, places=2) + delta = dst_info.modified - src_info.modified + self.assertAlmostEqual(delta.total_seconds(), 0, places=2) @unittest.skipUnless(osfs.sendfile, "sendfile not supported") @unittest.skipIf( From ee211479c7df1e5297bf929b1c93fab649ea783c Mon Sep 17 00:00:00 2001 From: atollk Date: Thu, 29 Apr 2021 01:08:39 +0200 Subject: [PATCH 199/309] Fixes from code review. --- fs/info.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/info.py b/fs/info.py index 6c6cdb0e..e7a4fd84 100644 --- a/fs/info.py +++ b/fs/info.py @@ -317,12 +317,9 @@ def modified(self): namespace is not in the Info. """ - try: - self._require_namespace("details") - return self._make_datetime(self.get("details", "modified")) - except MissingInfoNamespace: - self._require_namespace("modified") - return self._make_datetime(self.get("modified", "modified")) + namespace = "modified" if self.has_namespace("modified") else "details" + self._require_namespace(namespace) + return self._make_datetime(self.get(namespace, "modified")) @property def created(self): From 145bcea8d9a5572b6e99b6a13e20e75ffc667c9b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 27 Jun 2021 14:34:29 +0200 Subject: [PATCH 200/309] Remove the `supports_mtime` meta feature --- fs/base.py | 4 ---- fs/info.py | 6 +++--- fs/osfs.py | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/fs/base.py b/fs/base.py index 55edf3a4..82a5978d 100644 --- a/fs/base.py +++ b/fs/base.py @@ -712,8 +712,6 @@ def getmodified(self, path): it might only have limited accuracy. """ - if self.getmeta().get("supports_mtime", False): - return self.getinfo(path, namespaces=["modified"]).modified return self.getinfo(path, namespaces=["details"]).modified def getmeta(self, namespace="standard"): @@ -753,8 +751,6 @@ def getmeta(self, namespace="standard"): read_only `True` if this filesystem is read only. supports_rename `True` if this filesystem supports an `os.rename` operation. - supports_mtime `True` if this filesystem supports a native - operation to retrieve the "last modified" time. =================== ============================================ Most builtin filesystems will provide all these keys, and third- diff --git a/fs/info.py b/fs/info.py index e7a4fd84..03bf27cd 100644 --- a/fs/info.py +++ b/fs/info.py @@ -317,9 +317,9 @@ def modified(self): namespace is not in the Info. """ - namespace = "modified" if self.has_namespace("modified") else "details" - self._require_namespace(namespace) - return self._make_datetime(self.get(namespace, "modified")) + self._require_namespace("details") + _time = self._make_datetime(self.get("details", "modified")) + return _time @property def created(self): diff --git a/fs/osfs.py b/fs/osfs.py index c0de12cd..ac43471a 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -145,7 +145,6 @@ def __init__( "network": False, "read_only": False, "supports_rename": True, - "supports_mtime": False, "thread_safe": True, "unicode_paths": os.path.supports_unicode_filenames, "virtual": False, From 2d9b6375d6d3958bd27615b8ff83594bf062b40b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 27 Jun 2021 14:48:30 +0200 Subject: [PATCH 201/309] Overload `FTPFS.getmodified` to use the MDTM feature if available --- fs/ftpfs.py | 38 +++++++++++++++++++------------------- tests/test_ftpfs.py | 3 ++- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 260592ce..9f480625 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -42,6 +42,7 @@ from .path import basename from .path import normpath from .path import split +from .time import epoch_to_datetime from . import _ftp_parse as ftp_parse if typing.TYPE_CHECKING: @@ -574,6 +575,12 @@ def supports_mlst(self): """bool: whether the server supports MLST feature.""" return "MLST" in self.features + @property + def supports_mdtm(self): + # type: () -> bool + """bool: whether the server supports the MDTM feature.""" + return "MDTM" in self.features + def create(self, path, wipe=False): # type: (Text, bool) -> bool _path = self.validatepath(path) @@ -669,25 +676,6 @@ def getinfo(self, path, namespaces=None): } ) - if "modified" in namespaces: - if "details" in namespaces: - warnings.warn( - "FTPFS.getinfo called with both 'modified' and 'details'" - " namespace. The former will be ignored.", - UserWarning, - ) - else: - with self._lock: - with ftp_errors(self, path=path): - cmd = "MDTM " + _encode( - self.validatepath(path), self.ftp.encoding - ) - response = self.ftp.sendcmd(cmd) - modified_info = { - "modified": self._parse_ftp_time(response.split()[1]) - } - return Info({"modified": modified_info}) - if self.supports_mlst: with self._lock: with ftp_errors(self, path=path): @@ -716,6 +704,18 @@ def getmeta(self, namespace="standard"): _meta["supports_mtime"] = "MDTM" in self.features return _meta + def getmodified(self, path): + # type: (Text) -> Optional[datetime] + if self.supports_mdtm: + _path = self.validatepath(path) + with self._lock: + with ftp_errors(self, path=path): + cmd = "MDTM " + _encode(_path, self.ftp.encoding) + response = self.ftp.sendcmd(cmd) + mtime = self._parse_ftp_time(response.split()[1]) + return epoch_to_datetime(mtime) + return super().getmodified(self, path) + def listdir(self, path): # type: (Text) -> List[Text] _path = self.validatepath(path) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 0501589e..935582ff 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -265,7 +265,8 @@ def test_getinfo_modified(self): self.assertIn("MDTM", self.fs.features) self.fs.create("bar") mtime_detail = self.fs.getinfo("bar", ("basic", "details")).modified - mtime_modified = self.fs.getinfo("bar", ("modified",)).modified + mtime_modified = self.fs.getmodified("bar") + print(mtime_detail, mtime_modified) # Microsecond and seconds might not actually be supported by all # FTP commands, so we strip them before comparing if it looks # like at least one of the two values does not contain them. From 5a67c8a1bcd48d6c0153ec4ed58b2a256cf1c162 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 27 Jun 2021 14:56:30 +0200 Subject: [PATCH 202/309] Fix type annotations and unused imports in `fs.ftpfs` --- fs/ftpfs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index 9f480625..b7a49988 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -12,12 +12,10 @@ import socket import threading import typing -import warnings from collections import OrderedDict from contextlib import contextmanager from ftplib import FTP - try: from ftplib import FTP_TLS except ImportError as err: @@ -705,7 +703,7 @@ def getmeta(self, namespace="standard"): return _meta def getmodified(self, path): - # type: (Text) -> Optional[datetime] + # type: (Text) -> Optional[datetime.datetime] if self.supports_mdtm: _path = self.validatepath(path) with self._lock: @@ -714,7 +712,7 @@ def getmodified(self, path): response = self.ftp.sendcmd(cmd) mtime = self._parse_ftp_time(response.split()[1]) return epoch_to_datetime(mtime) - return super().getmodified(self, path) + return super(FTPFS, self).getmodified(path) def listdir(self, path): # type: (Text) -> List[Text] From 4feb2425bc220716f0beb2fa313c9ebfc1ec65e1 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sat, 3 Jul 2021 11:53:59 +0200 Subject: [PATCH 203/309] Remove debug print from `tests.test_ftpfs` --- tests/test_ftpfs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index 935582ff..d4143aa0 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -266,7 +266,6 @@ def test_getinfo_modified(self): self.fs.create("bar") mtime_detail = self.fs.getinfo("bar", ("basic", "details")).modified mtime_modified = self.fs.getmodified("bar") - print(mtime_detail, mtime_modified) # Microsecond and seconds might not actually be supported by all # FTP commands, so we strip them before comparing if it looks # like at least one of the two values does not contain them. From baa05606487d7aad2b7be5dd42a33276d463e4d1 Mon Sep 17 00:00:00 2001 From: Will McGugan Date: Wed, 29 Sep 2021 15:04:13 +0100 Subject: [PATCH 204/309] Update FUNDING.yml --- .github/FUNDING.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 6762ecbc..fd623f58 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,5 @@ # These are supported funding model platforms -custom: willmcgugan +github: willmcgugan +ko_fi: willmcgugan +tidelift: "pypi/rich" From 19cfd868e774aae874782b6410274cbaf1d20d19 Mon Sep 17 00:00:00 2001 From: Ben Lindsay Date: Sun, 10 Oct 2021 15:44:33 -0500 Subject: [PATCH 205/309] Typo fix --- docs/source/guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/guide.rst b/docs/source/guide.rst index e1f078ee..e8921f59 100644 --- a/docs/source/guide.rst +++ b/docs/source/guide.rst @@ -176,7 +176,7 @@ You can open a file from a FS object with :meth:`~fs.base.FS.open`, which is ver In the case of a ``OSFS``, a standard file-like object will be returned. Other filesystems may return a different object supporting the same methods. For instance, :class:`~fs.memoryfs.MemoryFS` will return a ``io.BytesIO`` object. -PyFilesystem also offers a number of shortcuts for common file related operations. For instance, :meth:`~fs.base.FS.readbytes` will return the file contents as a bytes, and :meth:`~fs.base.FS.readtext` will read unicode text. These methods is generally preferable to explicitly opening files, as the FS object may have an optimized implementation. +PyFilesystem also offers a number of shortcuts for common file related operations. For instance, :meth:`~fs.base.FS.readbytes` will return the file contents as a bytes, and :meth:`~fs.base.FS.readtext` will read unicode text. These methods are generally preferable to explicitly opening files, as the FS object may have an optimized implementation. Other *shortcut* methods are :meth:`~fs.base.FS.download`, :meth:`~fs.base.FS.upload`, :meth:`~fs.base.FS.writebytes`, :meth:`~fs.base.FS.writetext`. From 2e62f2abce67415338f2bb12406020e5da23c71a Mon Sep 17 00:00:00 2001 From: Ben Lindsay Date: Sun, 10 Oct 2021 17:56:19 -0500 Subject: [PATCH 206/309] Typo fix --- docs/source/guide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/guide.rst b/docs/source/guide.rst index e8921f59..2c54655c 100644 --- a/docs/source/guide.rst +++ b/docs/source/guide.rst @@ -176,7 +176,7 @@ You can open a file from a FS object with :meth:`~fs.base.FS.open`, which is ver In the case of a ``OSFS``, a standard file-like object will be returned. Other filesystems may return a different object supporting the same methods. For instance, :class:`~fs.memoryfs.MemoryFS` will return a ``io.BytesIO`` object. -PyFilesystem also offers a number of shortcuts for common file related operations. For instance, :meth:`~fs.base.FS.readbytes` will return the file contents as a bytes, and :meth:`~fs.base.FS.readtext` will read unicode text. These methods are generally preferable to explicitly opening files, as the FS object may have an optimized implementation. +PyFilesystem also offers a number of shortcuts for common file related operations. For instance, :meth:`~fs.base.FS.readbytes` will return the file contents as bytes, and :meth:`~fs.base.FS.readtext` will read unicode text. These methods are generally preferable to explicitly opening files, as the FS object may have an optimized implementation. Other *shortcut* methods are :meth:`~fs.base.FS.download`, :meth:`~fs.base.FS.upload`, :meth:`~fs.base.FS.writebytes`, :meth:`~fs.base.FS.writetext`. From 1ef7ca93beac0524a316993285b8cf0f29372d12 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:07:09 +0100 Subject: [PATCH 207/309] Note that `OSFS.geturl` supports the `fs` purpose in `FS.geturl` --- fs/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index 82a5978d..83c36218 100644 --- a/fs/base.py +++ b/fs/base.py @@ -910,7 +910,8 @@ def geturl(self, path, purpose="download"): to retrieve for the given path (if there is more than one). The default is ``'download'``, which should return a URL that serves the file. Other filesystems may support - other values for ``purpose``. + other values for ``purpose``: for instance, `OSFS` supports + ``'fs'``, which returns a FS URL (see :ref:`fs-urls`). Returns: str: a URL. From 44d691f7ab4ae3ac9d3a5cab330d69a5da34a4c9 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:07:30 +0100 Subject: [PATCH 208/309] Make explicit that `SubFS` represents a sub-directory in a parent filesystem --- fs/subfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/subfs.py b/fs/subfs.py index 1357eb1f..0582734a 100644 --- a/fs/subfs.py +++ b/fs/subfs.py @@ -21,7 +21,7 @@ @six.python_2_unicode_compatible class SubFS(WrapFS[_F], typing.Generic[_F]): - """A sub-directory on another filesystem. + """A sub-directory on a parent filesystem. A SubFS is a filesystem object that maps to a sub-directory of another filesystem. This is the object that is returned by From 80774d0cbc7dca48ce62818cc5173ceb719e2cb7 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:17:01 +0100 Subject: [PATCH 209/309] Document the workaround for #485 in the `openers` page of the documentation --- docs/source/openers.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/source/openers.rst b/docs/source/openers.rst index 1fa643e5..b5a2a823 100644 --- a/docs/source/openers.rst +++ b/docs/source/openers.rst @@ -56,3 +56,25 @@ To open a filesysem with a FS URL, you can use :meth:`~fs.opener.registry.Regist from fs import open_fs projects_fs = open_fs('osfs://~/projects') + + +Manually registering Openers +---------------------------- + +The ``fs.opener`` registry uses an entry point to install external openers +(see :ref:`extension`), and it does so once, when you import `fs` for the first +time. In some rare cases where entry points are not available (for instance, +when running an embedded interpreter) or when extensions are installed *after* +the interpreter has started (for instance in a notebook, see +`PyFilesystem2#485 `_). + +However, a new opener can be installed manually at any time with the +`fs.opener.registry.install` method. For instance, here's how the opener for +the `s3fs `_ extension can be added to +the registry:: + + import fs.opener + from fs_s3fs.opener import S3FSOpener + + fs.opener.registry.install(S3FSOpener) + # fs.open_fs("s3fs://...") should now work From f2b6dfd019795d57dc2367172ecfbfaf5cdff884 Mon Sep 17 00:00:00 2001 From: Jon Hagg Date: Wed, 11 Aug 2021 16:06:58 -0700 Subject: [PATCH 210/309] Fix imports in `extension.rst` page of documentation --- docs/source/extension.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/extension.rst b/docs/source/extension.rst index 3180561b..c9dc3f69 100644 --- a/docs/source/extension.rst +++ b/docs/source/extension.rst @@ -29,7 +29,8 @@ Here's an example taken from an Amazon S3 Filesystem:: __all__ = ['S3FSOpener'] - from fs.opener import Opener, OpenerError + from fs.opener import Opener + from fs.opener.errors import OpenerError from ._s3fs import S3FS From ea32fbc751c52cc25633f52c6ab8ee7bceb330fb Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:27:24 +0100 Subject: [PATCH 211/309] Add @jon-hagg to the `CONTRIBUTORS.md` file --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 2f8ec34f..7fea14ca 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -17,6 +17,7 @@ Many thanks to the following developers for contributing to this project: - [George Macon](https://github.com/gmacon) - [Giampaolo Cimino](https://github.com/gpcimino) - [@Hoboneer](https://github.com/Hoboneer) +- [Jon Hagg](https://github.com/jon-hagg) - [Joseph Atkins-Turkish](https://github.com/Spacerat) - [Joshua Tauberer](https://github.com/JoshData) - [Justin Charlong](https://github.com/jcharlong) From 86c53707c274e4488e6750dafa588d6125d7d949 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:31:00 +0100 Subject: [PATCH 212/309] Add @benlindsay to the `CONTRIBUTORS.md` file --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7fea14ca..03462fc9 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,6 +6,7 @@ Many thanks to the following developers for contributing to this project: - [Andreas Tollkötter](https://github.com/atollk) - [Andrew Scheller](https://github.com/lurch) - [Andrey Serov](https://github.com/zmej-serow) +- [Ben Lindsay](https://github.com/benlindsay) - [Bernhard M. Wiedemann](https://github.com/bmwiedemann) - [@chfw](https://github.com/chfw) - [Dafna Hirschfeld](https://github.com/kamomil) From 6bea7d42e768b1274d1c0e2782b70bfab60f48a1 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:52:43 +0100 Subject: [PATCH 213/309] Add the `CONTRIBUTING.md` guide to the docs as a page --- docs/requirements.txt | 1 + docs/source/conf.py | 5 +++-- docs/source/contributing.md | 1 + docs/source/index.rst | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) create mode 120000 docs/source/contributing.md diff --git a/docs/requirements.txt b/docs/requirements.txt index c2d9a973..ee590c37 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ # the bare requirements for building docs Sphinx ~=3.0 sphinx-rtd-theme ~=0.5.1 +recommonmark ~=0.6 diff --git a/docs/source/conf.py b/docs/source/conf.py index 749c3330..2ec980f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -39,7 +39,8 @@ 'sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', - 'sphinx.ext.intersphinx' + 'sphinx.ext.intersphinx', + "recommonmark", ] # Add any paths that contain templates here, relative to this directory. @@ -63,7 +64,7 @@ # General information about the project. project = u'PyFilesystem' -copyright = u'2016-2017, Will McGugan' +copyright = u'2016-2021, Will McGugan and the PyFilesystem2 contributors' author = u'Will McGugan' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/source/contributing.md b/docs/source/contributing.md new file mode 120000 index 00000000..f939e75f --- /dev/null +++ b/docs/source/contributing.md @@ -0,0 +1 @@ +../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index c3fb2eb1..a2b72ebf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -25,7 +25,7 @@ Contents: external.rst interface.rst reference.rst - + contributing.md Indices and tables From 79a73679e5ce63db982a13d87feca13effc41daf Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 16 Nov 2021 16:54:45 +0100 Subject: [PATCH 214/309] Release v2.4.14 --- CHANGELOG.md | 11 +++++++---- fs/_version.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11d8330d..392fa744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,18 +8,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + +## [2.4.14] - 2021-11-16 + ### Added -- Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. +- Added `fs.copy.copy_file_if`, `fs.copy.copy_dir_if`, and `fs.copy.copy_fs_if`. Closes [#458](https://github.com/PyFilesystem/pyfilesystem2/issues/458). - Added `fs.base.FS.getmodified`. - + ### Changed - FTP servers that do not support the MLST command now try to use the MDTM command to - retrieve the last modification timestamp of a resource. + retrieve the last modification timestamp of a resource. Closes [#456](https://github.com/PyFilesystem/pyfilesystem2/pull/456). - + ### Fixed - Fixed performance bugs in `fs.copy.copy_dir_if_newer`. Test cases were adapted to catch those bugs in the future. diff --git a/fs/_version.py b/fs/_version.py index 8f74ece1..ecd8e424 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.13" +__version__ = "2.4.14" From 7f1963884dbb8c2a704a6951dab94aef9f4b26c0 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 13 Dec 2021 09:08:58 -0600 Subject: [PATCH 215/309] Fix parsing of FTP LIST command (#507) * fix parsing of FTP LIST command * update changelog and contributors --- CHANGELOG.md | 1 + CONTRIBUTORS.md | 1 + fs/_ftp_parse.py | 4 ++-- tests/test_ftp_parse.py | 26 ++++++++++++++++++++++++++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 392fa744..dab094b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +- Support more leniant usernames and group names in FTP servers [#507](https://github.com/PyFilesystem/pyfilesystem2/pull/507). Closes [#506](https://github.com/PyFilesystem/pyfilesystem2/pull/506). ## [2.4.14] - 2021-11-16 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 03462fc9..7e3fcbde 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -2,6 +2,7 @@ Many thanks to the following developers for contributing to this project: +- [Adrian Garcia Badaracco](https://github.com/adriangb) - [Alex Povel](https://github.com/alexpovel) - [Andreas Tollkötter](https://github.com/atollk) - [Andrew Scheller](https://github.com/lurch) diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index defc55ee..0061f875 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -24,9 +24,9 @@ \s+? (\d+) \s+? - ([\w\-]+) + ([A-Za-z0-9][A-Za-z0-9\-\.\_\@]*\$?) \s+? - ([\w\-]+) + ([A-Za-z0-9][A-Za-z0-9\-\.\_\@]*\$?) \s+? (\d+) \s+? diff --git a/tests/test_ftp_parse.py b/tests/test_ftp_parse.py index d027082d..bd967aed 100644 --- a/tests/test_ftp_parse.py +++ b/tests/test_ftp_parse.py @@ -50,7 +50,9 @@ def test_decode_linux(self, mock_localtime): drwxr-xr-x 12 0 0 4096 Sep 29 13:13 pub -rw-r--r-- 1 0 0 26 Mar 04 2010 robots.txt drwxr-xr-x 8 foo bar 4096 Oct 4 09:05 test + drwxr-xr-x 8 f b 4096 Oct 4 09:05 test drwxr-xr-x 2 foo-user foo-group 0 Jan 5 11:59 240485 + drwxr-xr-x 2 foo.user$ foo@group_ 0 Jan 5 11:59 240485 """ ) @@ -147,6 +149,18 @@ def test_decode_linux(self, mock_localtime): "ls": "drwxr-xr-x 8 foo bar 4096 Oct 4 09:05 test" }, }, + { + "access": { + "group": "b", + "permissions": ["g_r", "g_x", "o_r", "o_x", "u_r", "u_w", "u_x"], + "user": "f", + }, + "basic": {"is_dir": True, "name": "test"}, + "details": {"modified": 1507107900.0, "size": 4096, "type": 1}, + "ftp": { + "ls": "drwxr-xr-x 8 f b 4096 Oct 4 09:05 test" + }, + }, { "access": { "group": "foo-group", @@ -159,6 +173,18 @@ def test_decode_linux(self, mock_localtime): "ls": "drwxr-xr-x 2 foo-user foo-group 0 Jan 5 11:59 240485" }, }, + { + "access": { + "group": "foo@group_", + "permissions": ["g_r", "g_x", "o_r", "o_x", "u_r", "u_w", "u_x"], + "user": "foo.user$", + }, + "basic": {"is_dir": True, "name": "240485"}, + "details": {"modified": 1483617540.0, "size": 0, "type": 1}, + "ftp": { + "ls": "drwxr-xr-x 2 foo.user$ foo@group_ 0 Jan 5 11:59 240485" + }, + }, ] parsed = ftp_parse.parse(directory.strip().splitlines()) From 47f9cfd66232e6de2c0a997a3cf26962fc303040 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 16:18:58 +0100 Subject: [PATCH 216/309] Add missing `UnsupportedHash` exception to `fs.errors.__all__` --- fs/errors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/errors.py b/fs/errors.py index 2448c7a6..ba193bdb 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -51,6 +51,7 @@ "ResourceNotFound", "ResourceReadOnly", "Unsupported", + "UnsupportedHash", ] From fee7ee60b86cdb41be47d2f801d84ccaedfca95e Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 17:27:11 +0100 Subject: [PATCH 217/309] Update `fs.test` to make sure that files moved in the same FS get renamed --- fs/test.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/fs/test.py b/fs/test.py index 3e08f4ff..7be3f397 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1738,6 +1738,22 @@ def test_copy_dir_temp(self): self._test_copy_dir("temp://") self._test_copy_dir_write("temp://") + def test_move_dir_same_fs(self): + self.fs.makedirs("foo/bar/baz") + self.fs.makedir("egg") + self.fs.writetext("top.txt", "Hello, World") + self.fs.writetext("/foo/bar/baz/test.txt", "Goodbye, World") + + fs.move.move_dir(self.fs, "foo", self.fs, "foo2") + + expected = {"/egg", "/foo2", "/foo2/bar", "/foo2/bar/baz"} + self.assertEqual(set(walk.walk_dirs(self.fs)), expected) + self.assert_text("top.txt", "Hello, World") + self.assert_text("/foo2/bar/baz/test.txt", "Goodbye, World") + + self.assertEqual(sorted(self.fs.listdir("/")), ["egg", "foo2", "top.txt"]) + self.assertEqual(sorted(x.name for x in self.fs.scandir("/")), ["egg", "foo2", "top.txt"]) + def _test_move_dir_write(self, protocol): # Test moving to this filesystem from another. other_fs = open_fs(protocol) @@ -1760,19 +1776,6 @@ def test_move_dir_mem(self): def test_move_dir_temp(self): self._test_move_dir_write("temp://") - def test_move_same_fs(self): - self.fs.makedirs("foo/bar/baz") - self.fs.makedir("egg") - self.fs.writetext("top.txt", "Hello, World") - self.fs.writetext("/foo/bar/baz/test.txt", "Goodbye, World") - - fs.move.move_dir(self.fs, "foo", self.fs, "foo2") - - expected = {"/egg", "/foo2", "/foo2/bar", "/foo2/bar/baz"} - self.assertEqual(set(walk.walk_dirs(self.fs)), expected) - self.assert_text("top.txt", "Hello, World") - self.assert_text("/foo2/bar/baz/test.txt", "Goodbye, World") - def test_move_file_same_fs(self): text = "Hello, World" self.fs.makedir("foo").writetext("test.txt", text) @@ -1782,6 +1785,9 @@ def test_move_file_same_fs(self): self.assert_not_exists("foo/test.txt") self.assert_text("foo/test2.txt", text) + self.assertEqual(self.fs.listdir("foo"), ["test2.txt"]) + self.assertEqual(next(self.fs.scandir("foo")).name, "test2.txt") + def _test_move_file(self, protocol): other_fs = open_fs(protocol) From 13f1c293a7c579dad31011440b9e986b0148397f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 17:45:19 +0100 Subject: [PATCH 218/309] Fix `MemoryFS` entries not being renamed when moved --- fs/memoryfs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/memoryfs.py b/fs/memoryfs.py index c34d60a7..35ef0f51 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -463,8 +463,11 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): elif not overwrite and dst_name in dst_dir_entry: raise errors.DestinationExists(dst_path) + # move the entry from the src folder to the dst folder dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) + # make sure to update the entry name itself (see #509) + src_entry.name = dst_name if preserve_time: copy_modified_time(self, src_path, self, dst_path) @@ -481,12 +484,16 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): if not src_entry.is_dir: raise errors.DirectoryExpected(src_path) + # move the entry from the src folder to the dst folder dst_dir_entry = self._get_dir_entry(dst_dir) if dst_dir_entry is None or (not create and dst_name not in dst_dir_entry): raise errors.ResourceNotFound(dst_path) + # move the entry from the src folder to the dst folder dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) + # make sure to update the entry name itself (see #509) + src_entry.name = dst_name if preserve_time: copy_modified_time(self, src_path, self, dst_path) From c29a917fac30aa3842be49a6f83bac319f863c6a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 17:56:08 +0100 Subject: [PATCH 219/309] Reformat code of `fs.test` module with `black` --- fs/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/test.py b/fs/test.py index 7be3f397..0327f98d 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1752,7 +1752,9 @@ def test_move_dir_same_fs(self): self.assert_text("/foo2/bar/baz/test.txt", "Goodbye, World") self.assertEqual(sorted(self.fs.listdir("/")), ["egg", "foo2", "top.txt"]) - self.assertEqual(sorted(x.name for x in self.fs.scandir("/")), ["egg", "foo2", "top.txt"]) + self.assertEqual( + sorted(x.name for x in self.fs.scandir("/")), ["egg", "foo2", "top.txt"] + ) def _test_move_dir_write(self, protocol): # Test moving to this filesystem from another. From d76e7f3818a85577e7b21884404a8850177d67a5 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 18:14:00 +0100 Subject: [PATCH 220/309] Update `CHANGELOG.md` with changes from #510 --- CHANGELOG.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dab094b1..ab0d827c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,19 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased -- Support more leniant usernames and group names in FTP servers [#507](https://github.com/PyFilesystem/pyfilesystem2/pull/507). Closes [#506](https://github.com/PyFilesystem/pyfilesystem2/pull/506). +### Changed + +- Support more lenient usernames and group names in FTP servers + ([#507](https://github.com/PyFilesystem/pyfilesystem2/pull/507)). + Closes [#506](https://github.com/PyFilesystem/pyfilesystem2/issues/506). + +### Fixed + +- Fixed `MemoryFS.move` and `MemoryFS.movedir` not updating the name of moved + resources, causing `MemoryFS.scandir` to use the old name. + ([#510](https://github.com/PyFilesystem/pyfilesystem2/pull/510)). + Closes [#509](https://github.com/PyFilesystem/pyfilesystem2/issues/509). + ## [2.4.14] - 2021-11-16 From 84af56700e979198e51388b14b58c7c06e3fbdcd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 18:33:33 +0100 Subject: [PATCH 221/309] Update `test.yml` Actions workflow to run tests on Python 3.10 --- .github/workflows/test.yml | 9 +++++---- setup.cfg | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3b8a4932..ced43cd2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,7 @@ jobs: - 3.7 - 3.8 - 3.9 + - '3.10' - pypy-2.7 - pypy-3.6 - pypy-3.7 @@ -44,10 +45,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v1 - - name: Setup Python 3.9 + - name: Setup Python 3.10 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.10' - name: Install coverage package run: python -m pip install -U coverage - name: Download partial coverage reports @@ -76,10 +77,10 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v1 - - name: Setup Python 3.9 + - name: Setup Python '3.10' uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: '3.10' - name: Update pip run: python -m pip install -U pip wheel setuptools - name: Install tox diff --git a/setup.cfg b/setup.cfg index fcc75584..4c9a2f9a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,12 +21,12 @@ classifiers = Operating System :: OS Independent Programming Language :: Python Programming Language :: Python :: 2.7 - Programming Language :: Python :: 3.4 Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy Topic :: System :: Filesystems From a6ea045e766c76bae2e2fde19294c8275773ffde Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 13 Dec 2021 18:35:32 +0100 Subject: [PATCH 222/309] Update `tox` configuration in `setup.cfg` to run tests on Python 3.10 --- setup.cfg | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 4c9a2f9a..d6aa8943 100644 --- a/setup.cfg +++ b/setup.cfg @@ -130,7 +130,7 @@ markers = # --- Tox automation configuration --------------------------------------------- [tox:tox] -envlist = py{27,34}{,-scandir}, py{35,36,37,38,39}, pypy{27,36,37}, typecheck, codestyle, docstyle, codeformat +envlist = py{27,34}{,-scandir}, py{35,36,37,38,39,310}, pypy{27,36,37}, typecheck, codestyle, docstyle, codeformat sitepackages = false skip_missing_interpreters = true requires = @@ -141,9 +141,9 @@ commands = python -m coverage run --rcfile {toxinidir}/setup.cfg -m pytest {posa deps = -rtests/requirements.txt coverage~=5.0 - py{35,36,37,38,39,py36,py37}: pytest~=6.0 + py{35,36,37,38,39,310,py36,py37}: pytest~=6.0 py{27,34,py27}: pytest~=4.6 - py{35,36,37,38,39,py36,py37}: pytest-randomly~=3.5 + py{35,36,37,38,39,310,py36,py37}: pytest-randomly~=3.5 py{27,34,py27}: pytest-randomly~=1.2 scandir: .[scandir] !scandir: . @@ -183,6 +183,7 @@ python = 3.7: py37 3.8: py38 3.9: py39 + 3.10: py310 pypy-2.7: pypy27 pypy-3.6: pypy36 pypy-3.7: pypy37 From 3f6009bec00da677a4aede3ddb3966ebd7be28c1 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 21 Dec 2021 14:00:37 +0100 Subject: [PATCH 223/309] Update `move` and `movedir` methods of `WrapFS` to use the delegate FS methods --- fs/wrapfs.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 00edd7af..c0273bb8 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -169,24 +169,21 @@ def makedir( def move(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None - # A custom move permits a potentially optimized code path - src_fs, _src_path = self.delegate_path(src_path) - dst_fs, _dst_path = self.delegate_path(dst_path) + _fs, _src_path = self.delegate_path(src_path) + _, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): - if not overwrite and dst_fs.exists(_dst_path): - raise errors.DestinationExists(_dst_path) - move_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) + _fs.move( + _src_path, _dst_path, overwrite=overwrite, preserve_time=preserve_time + ) def movedir(self, src_path, dst_path, create=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None - src_fs, _src_path = self.delegate_path(src_path) - dst_fs, _dst_path = self.delegate_path(dst_path) + _fs, _src_path = self.delegate_path(src_path) + _, _dst_path = self.delegate_path(dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): - if not create and not dst_fs.exists(_dst_path): - raise errors.ResourceNotFound(dst_path) - if not src_fs.getinfo(_src_path).is_dir: - raise errors.DirectoryExpected(src_path) - move_dir(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) + _fs.movedir( + _src_path, _dst_path, create=create, preserve_time=preserve_time + ) def openbin(self, path, mode="r", buffering=-1, **options): # type: (Text, Text, int, **Any) -> BinaryIO From f257c8c27612304db65b69427433b139b20b320f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 21 Dec 2021 14:09:53 +0100 Subject: [PATCH 224/309] Update `CHANGELOG.md` with fix from #511 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0d827c..df604fcb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/). resources, causing `MemoryFS.scandir` to use the old name. ([#510](https://github.com/PyFilesystem/pyfilesystem2/pull/510)). Closes [#509](https://github.com/PyFilesystem/pyfilesystem2/issues/509). +- Make `WrapFS.move` and `WrapFS.movedir` use the delegate FS methods instead + of `fs.move` functions, which was causing optimized implementation of + `movedir` to be always skipped. + ([#511](https://github.com/PyFilesystem/pyfilesystem2/pull/511)). ## [2.4.14] - 2021-11-16 From df3672618e02e080b78a1c6861a32544c7c85441 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Tue, 21 Dec 2021 14:19:05 +0100 Subject: [PATCH 225/309] Remove unused imports in `fs.wrapfs` --- fs/wrapfs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index c0273bb8..00984e72 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -11,7 +11,6 @@ from .base import FS from .copy import copy_file, copy_dir from .info import Info -from .move import move_file, move_dir from .path import abspath, join, normpath from .error_tools import unwrap_errors From 84dd668632110539f4386e23fcf79fb7a67e506f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 7 Feb 2022 13:21:04 +0100 Subject: [PATCH 226/309] Update `package.yml` to create a new release on GitHub --- .github/workflows/package.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 9afb05be..6e6903a4 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -129,3 +129,17 @@ jobs: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} skip_existing: false + + release: + environment: GitHub Releases + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/v')" + name: Release + needs: upload + steps: + - name: Checkout code + uses: actions/checkout@v1 + - name: Release a Changelog + uses: rasmus-saks/release-a-changelog-action@v1.0.1 + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' From be95b73636eaecce3fc917c1f1331c6d5a8aa38a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 7 Feb 2022 13:21:48 +0100 Subject: [PATCH 227/309] Release v2.4.15 --- CHANGELOG.md | 3 +++ fs/_version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df604fcb..917e0ef5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + +## [2.4.15] - 2022-02-07 + ### Changed - Support more lenient usernames and group names in FTP servers diff --git a/fs/_version.py b/fs/_version.py index ecd8e424..188c0e14 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.14" +__version__ = "2.4.15" From db837c571a8477909b13cfdd8945a0be772dac29 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Tue, 8 Feb 2022 13:16:04 +0100 Subject: [PATCH 228/309] add optimization to move --- fs/move.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/fs/move.py b/fs/move.py index 56e9e5ca..58fb7e73 100644 --- a/fs/move.py +++ b/fs/move.py @@ -4,11 +4,15 @@ from __future__ import print_function from __future__ import unicode_literals +from os.path import commonpath import typing +from . import open_fs, errors from .copy import copy_dir from .copy import copy_file from .opener import manage_fs +from .osfs import OSFS +from .path import frombase if typing.TYPE_CHECKING: from .base import FS @@ -55,6 +59,21 @@ def move_file( resources (defaults to `False`). """ + # optimization for moving files between different OSFS instances + if isinstance(src_fs, OSFS) and isinstance(dst_fs, OSFS): + try: + src_syspath = src_fs.getsyspath(src_path) + dst_syspath = dst_fs.getsyspath(dst_path) + common = commonpath([src_syspath, dst_syspath]) + rel_src = frombase(common, src_syspath) + rel_dst = frombase(common, dst_syspath) + with open_fs(common, writeable=True) as base: + base.move(rel_src, rel_dst, preserve_time=preserve_time) + return + except (ValueError, errors.NoSysPath): + # optimization cannot be applied + pass + with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: if _src_fs is _dst_fs: From d6db46084987c28616f8463996d75c42255b4ab7 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Tue, 8 Feb 2022 13:18:50 +0100 Subject: [PATCH 229/309] update contributors --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7e3fcbde..d1820428 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -38,6 +38,7 @@ Many thanks to the following developers for contributing to this project: - [Silvan Spross](https://github.com/sspross) - [@sqwishy](https://github.com/sqwishy) - [Sven Schliesing](https://github.com/muffl0n) +- [Thomas Feldmann](https://github.com/tfeldmann) - [Tim Gates](https://github.com/timgates42/) - [@tkossak](https://github.com/tkossak) - [Todd Levi](https://github.com/televi) From 43dc890d70064f0e81ef4c05b1fe4bbf9f8818f8 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Tue, 8 Feb 2022 13:32:00 +0100 Subject: [PATCH 230/309] add test --- tests/test_move.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_move.py b/tests/test_move.py index 6b12b2b6..2bae7340 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -6,6 +6,7 @@ import fs.move from fs import open_fs +from fs.path import join @parameterized_class(("preserve_time",), [(True,), (False,)]) @@ -37,6 +38,17 @@ def test_move_fs(self): self.assertEqual(dst_file1_info.modified, src_file1_info.modified) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + def test_move_file(self): + with open_fs("temp://") as temp: + syspath = temp.getsyspath("/") + a = open_fs(syspath) + a.makedir("dir") + b = open_fs(join(syspath, "dir")) + b.writetext("file.txt", "Content") + fs.move.move_file(b, "file.txt", a, "here.txt") + self.assertEqual(a.readtext("here.txt"), "Content") + self.assertFalse(b.exists("file.txt")) + def test_move_dir(self): namespaces = ("details", "modified") From 8c1875051b34c26597633218ca90ac24db6e766a Mon Sep 17 00:00:00 2001 From: Matthew Gamble Date: Wed, 23 Feb 2022 05:54:19 +1100 Subject: [PATCH 231/309] Don't fail fast when running tests (#520) This avoids the need to play whack-a-mole with CI failures, like I had to deal with on #518. Everything should be laid out on the table once. This saves everyone time. --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ced43cd2..1f67ecfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: - 2.7 From 196b5f54d8990c0a459faf7d78ce9e4e7bc8b03b Mon Sep 17 00:00:00 2001 From: Matthew Gamble Date: Sun, 13 Mar 2022 22:04:16 +1100 Subject: [PATCH 232/309] Remove pytz dependency (#518) * Remove pytz Everything that pytz was being utilised for can be serviced perfectly fine by the standard library. Meanwhile, pytz is known as a footgun: https://blog.ganssle.io/articles/2018/03/pytz-fastest-footgun.html * Introduce compatibility timezone code for python2 --- CHANGELOG.md | 2 ++ fs/_ftp_parse.py | 11 +++++++---- fs/_tzcompat.py | 30 ++++++++++++++++++++++++++++++ fs/test.py | 12 ++++++++---- fs/time.py | 15 ++++++++------- setup.cfg | 1 - tests/test_info.py | 17 ++++++++++------- tests/test_time.py | 11 +++++++---- 8 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 fs/_tzcompat.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 917e0ef5..cfc65ec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Support more lenient usernames and group names in FTP servers ([#507](https://github.com/PyFilesystem/pyfilesystem2/pull/507)). Closes [#506](https://github.com/PyFilesystem/pyfilesystem2/issues/506). +- Removed dependency on pytz ([#518](https://github.com/PyFilesystem/pyfilesystem2/pull/518)). + Closes [#516](https://github.com/PyFilesystem/pyfilesystem2/issues/518). ### Fixed diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 0061f875..42c0720e 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -3,17 +3,20 @@ from __future__ import unicode_literals import unicodedata -import datetime import re import time +from datetime import datetime -from pytz import UTC +try: + from datetime import timezone +except ImportError: + from ._tzcompat import timezone # type: ignore from .enums import ResourceType from .permissions import Permissions -EPOCH_DT = datetime.datetime.fromtimestamp(0, UTC) +EPOCH_DT = datetime.fromtimestamp(0, timezone.utc) RE_LINUX = re.compile( @@ -98,7 +101,7 @@ def _parse_time(t, formats): day = _t.tm_mday hour = _t.tm_hour minutes = _t.tm_min - dt = datetime.datetime(year, month, day, hour, minutes, tzinfo=UTC) + dt = datetime(year, month, day, hour, minutes, tzinfo=timezone.utc) epoch_time = (dt - EPOCH_DT).total_seconds() return epoch_time diff --git a/fs/_tzcompat.py b/fs/_tzcompat.py new file mode 100644 index 00000000..282859b2 --- /dev/null +++ b/fs/_tzcompat.py @@ -0,0 +1,30 @@ +"""Compatibility shim for python2's lack of datetime.timezone. + +This is the example code from the Python 2 documentation: +https://docs.python.org/2.7/library/datetime.html#tzinfo-objects +""" + +from datetime import tzinfo, timedelta + + +ZERO = timedelta(0) + + +class UTC(tzinfo): + """UTC""" + + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return ZERO + + +utc = UTC() + + +class timezone: + utc = utc diff --git a/fs/test.py b/fs/test.py index 0327f98d..0968ce75 100644 --- a/fs/test.py +++ b/fs/test.py @@ -26,7 +26,6 @@ from fs.opener import open_fs from fs.subfs import ClosingSubFS, SubFS -import pytz import six from six import text_type @@ -35,6 +34,11 @@ else: import collections.abc as collections_abc +try: + from datetime import timezone +except ImportError: + from ._tzcompat import timezone # type: ignore + UNICODE_TEXT = """ @@ -1196,9 +1200,9 @@ def test_settimes(self): can_write_acccess = info.is_writeable("details", "accessed") can_write_modified = info.is_writeable("details", "modified") if can_write_acccess: - self.assertEqual(info.accessed, datetime(2016, 7, 5, tzinfo=pytz.UTC)) + self.assertEqual(info.accessed, datetime(2016, 7, 5, tzinfo=timezone.utc)) if can_write_modified: - self.assertEqual(info.modified, datetime(2016, 7, 5, tzinfo=pytz.UTC)) + self.assertEqual(info.modified, datetime(2016, 7, 5, tzinfo=timezone.utc)) def test_touch(self): self.fs.touch("new.txt") @@ -1206,7 +1210,7 @@ def test_touch(self): self.fs.settimes("new.txt", datetime(2016, 7, 5)) info = self.fs.getinfo("new.txt", namespaces=["details"]) if info.is_writeable("details", "accessed"): - self.assertEqual(info.accessed, datetime(2016, 7, 5, tzinfo=pytz.UTC)) + self.assertEqual(info.accessed, datetime(2016, 7, 5, tzinfo=timezone.utc)) now = time.time() self.fs.touch("new.txt") accessed = self.fs.getinfo("new.txt", namespaces=["details"]).raw[ diff --git a/fs/time.py b/fs/time.py index cdf06061..52a59a77 100644 --- a/fs/time.py +++ b/fs/time.py @@ -7,17 +7,16 @@ import typing from calendar import timegm from datetime import datetime -from pytz import UTC, timezone + +try: + from datetime import timezone +except ImportError: + from ._tzcompat import timezone # type: ignore if typing.TYPE_CHECKING: from typing import Optional -utcfromtimestamp = datetime.utcfromtimestamp -utclocalize = UTC.localize -GMT = timezone("GMT") - - def datetime_to_epoch(d): # type: (datetime) -> int """Convert datetime to epoch.""" @@ -39,4 +38,6 @@ def epoch_to_datetime(t): # noqa: D103 def epoch_to_datetime(t): # type: (Optional[int]) -> Optional[datetime] """Convert epoch time to a UTC datetime.""" - return utclocalize(utcfromtimestamp(t)) if t is not None else None + if t is None: + return None + return datetime.fromtimestamp(t, tz=timezone.utc) diff --git a/setup.cfg b/setup.cfg index d6aa8943..06dac8f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,7 +43,6 @@ setup_requires = setuptools >=38.3.0 install_requires = appdirs~=1.4.3 - pytz setuptools six ~=1.10 enum34 ~=1.1.6 ; python_version < '3.4' diff --git a/tests/test_info.py b/tests/test_info.py index f83c1e7b..f6d6cfe9 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,15 +1,18 @@ from __future__ import unicode_literals -import datetime +from datetime import datetime import unittest -import pytz - from fs.enums import ResourceType from fs.info import Info from fs.permissions import Permissions from fs.time import datetime_to_epoch +try: + from datetime import timezone +except ImportError: + from fs._tzcompat import timezone # type: ignore + class TestInfo(unittest.TestCase): def test_empty(self): @@ -71,10 +74,10 @@ def test_basic(self): def test_details(self): dates = [ - datetime.datetime(2016, 7, 5, tzinfo=pytz.UTC), - datetime.datetime(2016, 7, 6, tzinfo=pytz.UTC), - datetime.datetime(2016, 7, 7, tzinfo=pytz.UTC), - datetime.datetime(2016, 7, 8, tzinfo=pytz.UTC), + datetime(2016, 7, 5, tzinfo=timezone.utc), + datetime(2016, 7, 6, tzinfo=timezone.utc), + datetime(2016, 7, 7, tzinfo=timezone.utc), + datetime(2016, 7, 8, tzinfo=timezone.utc), ] epochs = [datetime_to_epoch(d) for d in dates] diff --git a/tests/test_time.py b/tests/test_time.py index a6bf5587..5933e888 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -3,18 +3,21 @@ from datetime import datetime import unittest -import pytz - from fs.time import datetime_to_epoch, epoch_to_datetime +try: + from datetime import timezone +except ImportError: + from fs._tzcompat import timezone # type: ignore + class TestEpoch(unittest.TestCase): def test_epoch_to_datetime(self): self.assertEqual( - epoch_to_datetime(142214400), datetime(1974, 7, 5, tzinfo=pytz.UTC) + epoch_to_datetime(142214400), datetime(1974, 7, 5, tzinfo=timezone.utc) ) def test_datetime_to_epoch(self): self.assertEqual( - datetime_to_epoch(datetime(1974, 7, 5, tzinfo=pytz.UTC)), 142214400 + datetime_to_epoch(datetime(1974, 7, 5, tzinfo=timezone.utc)), 142214400 ) From 863f7e4a5bdae3c478704e3104dc36ce0d93502b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 12:25:40 +0100 Subject: [PATCH 233/309] Add @djmattyg007 to the list of contributors --- CONTRIBUTORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 7e3fcbde..173f6a9d 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -27,6 +27,7 @@ Many thanks to the following developers for contributing to this project: - [Martin Durant](https://github.com/martindurant) - [Martin Larralde](https://github.com/althonos) - [Masaya Nakamura](https://github.com/mashabow) +- [Matthew Gamble](https://github.com/djmattyg007) - [Morten Engelhardt Olsen](https://github.com/xoriath) - [@mrg0029](https://github.com/mrg0029) - [Nathan Goldbaum](https://github.com/ngoldbaum) From a5954e4e0dcd06b61338186beeda5b19f7187a37 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Sun, 13 Mar 2022 19:27:14 +0800 Subject: [PATCH 234/309] Use `platformdirs` instead of `appdirs` (#489) * Use platformdirs instead of appdirs Use the better maintained `platformdirs` instead of `appdirs`. * Add @felixonmars to the list of contributors * Update `CHANGELOG.md` with changes from #489 Co-authored-by: Martin Larralde --- CHANGELOG.md | 5 +++++ CONTRIBUTORS.md | 1 + fs/appfs.py | 6 +++--- setup.cfg | 2 +- tests/test_appfs.py | 2 +- tests/test_opener.py | 6 +++--- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfc65ec4..5dfad690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### Changed + +- Replaced `appdirs` with `platformdirs` dependency + ([#489](https://github.com/PyFilesystem/pyfilesystem2/pull/489)). + ## [2.4.15] - 2022-02-07 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 173f6a9d..247b31cc 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -14,6 +14,7 @@ Many thanks to the following developers for contributing to this project: - [Diego Argueta](https://github.com/dargueta) - [Eelke van den Bos](https://github.com/eelkevdbos) - [Egor Namakonov](https://github.com/fresheed) +- [Felix Yan](https://github.com/felixonmars) - [@FooBarQuaxx](https://github.com/FooBarQuaxx) - [Geoff Jukes](https://github.com/geoffjukes) - [George Macon](https://github.com/gmacon) diff --git a/fs/appfs.py b/fs/appfs.py index 131ea8a8..4c337e44 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -5,7 +5,7 @@ subclasses of `~fs.osfs.OSFS`. """ -# Thanks to authors of https://pypi.org/project/appdirs +# Thanks to authors of https://pypi.org/project/platformdirs # see http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx @@ -16,7 +16,7 @@ from .osfs import OSFS from ._repr import make_repr -from appdirs import AppDirs +from platformdirs import PlatformDirs if typing.TYPE_CHECKING: from typing import Optional, Text @@ -78,7 +78,7 @@ def __init__( will be created if it does not exist. """ - self.app_dirs = AppDirs(appname, author, version, roaming) + self.app_dirs = PlatformDirs(appname, author, version, roaming) self._create = create super(_AppFS, self).__init__( getattr(self.app_dirs, self.app_dir), create=create diff --git a/setup.cfg b/setup.cfg index 06dac8f0..cd79fe78 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ packages = find: setup_requires = setuptools >=38.3.0 install_requires = - appdirs~=1.4.3 + platformdirs~=2.0.2 setuptools six ~=1.10 enum34 ~=1.1.6 ; python_version < '3.4' diff --git a/tests/test_appfs.py b/tests/test_appfs.py index acc8a7f7..8cf4b9eb 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -30,7 +30,7 @@ def tearDownClass(cls): def make_fs(self): with mock.patch( - "appdirs.{}".format(self.AppFS.app_dir), + "platformdirs.{}".format(self.AppFS.app_dir), autospec=True, spec_set=True, return_value=tempfile.mkdtemp(dir=self.tmpdir), diff --git a/tests/test_opener.py b/tests/test_opener.py index bc2f5cd7..49b1a50d 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -264,7 +264,7 @@ def test_open_fs(self): mem_fs_2 = opener.open_fs(mem_fs) self.assertEqual(mem_fs, mem_fs_2) - @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_open_userdata(self, app_dir): app_dir.return_value = self.tmpdir @@ -276,7 +276,7 @@ def test_open_userdata(self, app_dir): self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, "1.0") - @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_open_userdata_no_version(self, app_dir): app_dir.return_value = self.tmpdir @@ -285,7 +285,7 @@ def test_open_userdata_no_version(self, app_dir): self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, None) - @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_user_data_opener(self, app_dir): app_dir.return_value = self.tmpdir From f253d9fab233f891ddb7a63016a019c49b7673a6 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 12:36:11 +0100 Subject: [PATCH 235/309] Fix `getmodified` summary as suggested by @lurch in #462 --- docs/source/interface.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/interface.rst b/docs/source/interface.rst index 0d67c0c6..924ac3fb 100644 --- a/docs/source/interface.rst +++ b/docs/source/interface.rst @@ -20,7 +20,7 @@ The following is a complete list of methods on PyFilesystem objects. * :meth:`~fs.base.FS.getdetails` Get details info namespace for a resource. * :meth:`~fs.base.FS.getinfo` Get info regarding a file or directory. * :meth:`~fs.base.FS.getmeta` Get meta information for a resource. -* :meth:`~fs.base.FS.getmodified` Get info regarding the last modified time of a resource. +* :meth:`~fs.base.FS.getmodified` Get the last modified time of a resource. * :meth:`~fs.base.FS.getospath` Get path with encoding expected by the OS. * :meth:`~fs.base.FS.getsize` Get the size of a file. * :meth:`~fs.base.FS.getsyspath` Get the system path of a resource, if one exists. From 8b7269716e85bbb62dd22fa35d1b25e29f3c358b Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 17:42:30 +0100 Subject: [PATCH 236/309] Update `ReadZipFS.openbin` handler to use native `zipfile` code (#527) * Remove compatibility code for making `_ZipExtFile` seekable starting from Python 3.7 * Mark `ReadZipFS` as a case-sensitive filesystem * Update `CHANGELOG.md` with changes from #527 --- CHANGELOG.md | 5 ++ fs/zipfs.py | 185 +++++++++++++++++++++++++++++++++------------------ 2 files changed, 124 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dfad690..249602dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Replaced `appdirs` with `platformdirs` dependency ([#489](https://github.com/PyFilesystem/pyfilesystem2/pull/489)). +- Make `fs.zipfs._ZipExtFile` use the seeking mechanism implemented + in the Python standard library in Python version 3.7 and later + ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). +- Mark `fs.zipfs.ReadZipFS` as a case-sensitive filesystem + ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). ## [2.4.15] - 2022-02-07 diff --git a/fs/zipfs.py b/fs/zipfs.py index 12d8a668..324fa58d 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -4,6 +4,7 @@ from __future__ import print_function from __future__ import unicode_literals +import sys import typing import zipfile from datetime import datetime @@ -51,74 +52,126 @@ def __init__(self, fs, name): # noqa: D107 self._pos = 0 super(_ZipExtFile, self).__init__(_zip.open(name), "r", name) - def read(self, size=-1): - # type: (int) -> bytes - buf = self._f.read(-1 if size is None else size) - self._pos += len(buf) - return buf - - def read1(self, size=-1): - # type: (int) -> bytes - buf = self._f.read1(-1 if size is None else size) # type: ignore - self._pos += len(buf) - return buf - - def seek(self, offset, whence=Seek.set): - # type: (int, SupportsInt) -> int - """Change stream position. - - Change the stream position to the given byte offset. The - offset is interpreted relative to the position indicated by - ``whence``. - - Arguments: - offset (int): the offset to the new position, in bytes. - whence (int): the position reference. Possible values are: - * `Seek.set`: start of stream (the default). - * `Seek.current`: current position; offset may be negative. - * `Seek.end`: end of stream; offset must be negative. - - Returns: - int: the new absolute position. - - Raises: - ValueError: when ``whence`` is not known, or ``offset`` - is invalid. - - Note: - Zip compression does not support seeking, so the seeking - is emulated. Seeking somewhere else than the current position - will need to either: - * reopen the file and restart decompression - * read and discard data to advance in the file - - """ - _whence = int(whence) - if _whence == Seek.current: - offset += self._pos - if _whence == Seek.current or _whence == Seek.set: - if offset < 0: - raise ValueError("Negative seek position {}".format(offset)) - elif _whence == Seek.end: - if offset > 0: - raise ValueError("Positive seek position {}".format(offset)) - offset += self._end - else: - raise ValueError( - "Invalid whence ({}, should be {}, {} or {})".format( - _whence, Seek.set, Seek.current, Seek.end + # NOTE(@althonos): Starting from Python 3.7, files inside a Zip archive are + # seekable provided they were opened from a seekable file + # handle. Before that, we can emulate a seek using the + # read method, although it adds a ton of overhead and is + # way less efficient than extracting once to a BytesIO. + if sys.version_info < (3, 7): + + def read(self, size=-1): + # type: (int) -> bytes + buf = self._f.read(-1 if size is None else size) + self._pos += len(buf) + return buf + + def read1(self, size=-1): + # type: (int) -> bytes + buf = self._f.read1(-1 if size is None else size) # type: ignore + self._pos += len(buf) + return buf + + def tell(self): + # type: () -> int + return self._pos + + def seekable(self): + return True + + def seek(self, offset, whence=Seek.set): + # type: (int, SupportsInt) -> int + """Change stream position. + + Change the stream position to the given byte offset. The + offset is interpreted relative to the position indicated by + ``whence``. + + Arguments: + offset (int): the offset to the new position, in bytes. + whence (int): the position reference. Possible values are: + * `Seek.set`: start of stream (the default). + * `Seek.current`: current position; offset may be negative. + * `Seek.end`: end of stream; offset must be negative. + + Returns: + int: the new absolute position. + + Raises: + ValueError: when ``whence`` is not known, or ``offset`` + is invalid. + + Note: + Zip compression does not support seeking, so the seeking + is emulated. Seeking somewhere else than the current position + will need to either: + * reopen the file and restart decompression + * read and discard data to advance in the file + + """ + _whence = int(whence) + if _whence == Seek.current: + offset += self._pos + if _whence == Seek.current or _whence == Seek.set: + if offset < 0: + raise ValueError("Negative seek position {}".format(offset)) + elif _whence == Seek.end: + if offset > 0: + raise ValueError("Positive seek position {}".format(offset)) + offset += self._end + else: + raise ValueError( + "Invalid whence ({}, should be {}, {} or {})".format( + _whence, Seek.set, Seek.current, Seek.end + ) ) - ) - if offset < self._pos: - self._f = self._zip.open(self.name) # type: ignore - self._pos = 0 - self.read(offset - self._pos) - return self._pos + if offset < self._pos: + self._f = self._zip.open(self.name) # type: ignore + self._pos = 0 + self.read(offset - self._pos) + return self._pos + + else: + + def seek(self, offset, whence=Seek.set): + # type: (int, SupportsInt) -> int + """Change stream position. + + Change the stream position to the given byte offset. The + offset is interpreted relative to the position indicated by + ``whence``. + + Arguments: + offset (int): the offset to the new position, in bytes. + whence (int): the position reference. Possible values are: + * `Seek.set`: start of stream (the default). + * `Seek.current`: current position; offset may be negative. + * `Seek.end`: end of stream; offset must be negative. + + Returns: + int: the new absolute position. + + Raises: + ValueError: when ``whence`` is not known, or ``offset`` + is invalid. + + """ + _whence = int(whence) + _pos = self.tell() + if _whence == Seek.current or _whence == Seek.set: + if _pos + offset < 0: + raise ValueError("Negative seek position {}".format(offset)) + elif _whence == Seek.end: + if _pos + offset > 0: + raise ValueError("Positive seek position {}".format(offset)) + else: + raise ValueError( + "Invalid whence ({}, should be {}, {} or {})".format( + _whence, Seek.set, Seek.current, Seek.end + ) + ) - def tell(self): - # type: () -> int - return self._pos + return self._f.seek(offset, _whence) class ZipFS(WrapFS): @@ -279,7 +332,7 @@ class ReadZipFS(FS): """A readable zip file.""" _meta = { - "case_insensitive": True, + "case_insensitive": False, "network": False, "read_only": True, "supports_rename": False, From 483423252f472a0be05ca6a52419a9b2b373804f Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Sun, 13 Mar 2022 19:57:35 +0100 Subject: [PATCH 237/309] Fix mechanism used to check for invalid `whence` in `fs.zipfs` --- fs/zipfs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fs/zipfs.py b/fs/zipfs.py index 324fa58d..5c03754c 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -158,11 +158,14 @@ def seek(self, offset, whence=Seek.set): """ _whence = int(whence) _pos = self.tell() - if _whence == Seek.current or _whence == Seek.set: + if _whence == Seek.set: + if offset < 0: + raise ValueError("Negative seek position {}".format(offset)) + elif _whence == Seek.current: if _pos + offset < 0: raise ValueError("Negative seek position {}".format(offset)) elif _whence == Seek.end: - if _pos + offset > 0: + if offset > 0: raise ValueError("Positive seek position {}".format(offset)) else: raise ValueError( From cb49b6e91ddc5ecc9fd591bdc3642dce54feef91 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Thu, 24 Mar 2022 16:54:12 +0100 Subject: [PATCH 238/309] optimization not dependent of OSFS anymore --- fs/move.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/fs/move.py b/fs/move.py index 58fb7e73..705fb63a 100644 --- a/fs/move.py +++ b/fs/move.py @@ -59,21 +59,6 @@ def move_file( resources (defaults to `False`). """ - # optimization for moving files between different OSFS instances - if isinstance(src_fs, OSFS) and isinstance(dst_fs, OSFS): - try: - src_syspath = src_fs.getsyspath(src_path) - dst_syspath = dst_fs.getsyspath(dst_path) - common = commonpath([src_syspath, dst_syspath]) - rel_src = frombase(common, src_syspath) - rel_dst = frombase(common, dst_syspath) - with open_fs(common, writeable=True) as base: - base.move(rel_src, rel_dst, preserve_time=preserve_time) - return - except (ValueError, errors.NoSysPath): - # optimization cannot be applied - pass - with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: if _src_fs is _dst_fs: @@ -81,6 +66,16 @@ def move_file( _src_fs.move( src_path, dst_path, overwrite=True, preserve_time=preserve_time ) + elif _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): + # if both filesystems have a syspath we create a new OSFS from a + # common parent folder and use it to move the file. + src_syspath = src_fs.getsyspath(src_path) + dst_syspath = dst_fs.getsyspath(dst_path) + common = commonpath([src_syspath, dst_syspath]) + rel_src = frombase(common, src_syspath) + rel_dst = frombase(common, dst_syspath) + with open_fs(common, writeable=True) as base: + base.move(rel_src, rel_dst, preserve_time=preserve_time) else: # Standard copy and delete with _src_fs.lock(), _dst_fs.lock(): From 448f15f7ff0e22c1a6cb90ba1e34538d85e3e0bd Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Thu, 24 Mar 2022 17:04:29 +0100 Subject: [PATCH 239/309] remove unneeded import --- fs/move.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/move.py b/fs/move.py index 705fb63a..bdc27415 100644 --- a/fs/move.py +++ b/fs/move.py @@ -7,11 +7,10 @@ from os.path import commonpath import typing -from . import open_fs, errors +from . import open_fs from .copy import copy_dir from .copy import copy_file from .opener import manage_fs -from .osfs import OSFS from .path import frombase if typing.TYPE_CHECKING: From 68846cea26f7e530b1f25002ba61a108914c8e1f Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Thu, 24 Mar 2022 17:09:24 +0100 Subject: [PATCH 240/309] lock src and dst for move operation --- fs/move.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/move.py b/fs/move.py index bdc27415..a5700c1b 100644 --- a/fs/move.py +++ b/fs/move.py @@ -73,7 +73,9 @@ def move_file( common = commonpath([src_syspath, dst_syspath]) rel_src = frombase(common, src_syspath) rel_dst = frombase(common, dst_syspath) - with open_fs(common, writeable=True) as base: + with _src_fs.lock(), _dst_fs.lock(), open_fs( + common, writeable=True + ) as base: base.move(rel_src, rel_dst, preserve_time=preserve_time) else: # Standard copy and delete From 003e1b7955c75fef6771a70eae79a876aaaa8b79 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Thu, 24 Mar 2022 18:33:07 +0100 Subject: [PATCH 241/309] handle ValueError --- fs/move.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/fs/move.py b/fs/move.py index a5700c1b..bc287eb2 100644 --- a/fs/move.py +++ b/fs/move.py @@ -58,6 +58,25 @@ def move_file( resources (defaults to `False`). """ + if _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): + # if both filesystems have a syspath we create a new OSFS from a + # common parent folder and use it to move the file. + with manage_fs(src_fs) as _src_fs: + with manage_fs(dst_fs, create=True) as _dst_fs: + try: + src_syspath = src_fs.getsyspath(src_path) + dst_syspath = dst_fs.getsyspath(dst_path) + common = commonpath([src_syspath, dst_syspath]) + rel_src = frombase(common, src_syspath) + rel_dst = frombase(common, dst_syspath) + with _src_fs.lock(), _dst_fs.lock(), open_fs( + common, writeable=True + ) as base: + base.move(rel_src, rel_dst, preserve_time=preserve_time) + return # optimization worked, exit early + except ValueError: + pass + with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: if _src_fs is _dst_fs: @@ -65,18 +84,6 @@ def move_file( _src_fs.move( src_path, dst_path, overwrite=True, preserve_time=preserve_time ) - elif _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): - # if both filesystems have a syspath we create a new OSFS from a - # common parent folder and use it to move the file. - src_syspath = src_fs.getsyspath(src_path) - dst_syspath = dst_fs.getsyspath(dst_path) - common = commonpath([src_syspath, dst_syspath]) - rel_src = frombase(common, src_syspath) - rel_dst = frombase(common, dst_syspath) - with _src_fs.lock(), _dst_fs.lock(), open_fs( - common, writeable=True - ) as base: - base.move(rel_src, rel_dst, preserve_time=preserve_time) else: # Standard copy and delete with _src_fs.lock(), _dst_fs.lock(): From fa1803b6a8fe13b1e629fd5abaccd0a23b71f1cb Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Thu, 24 Mar 2022 18:36:56 +0100 Subject: [PATCH 242/309] fix unbound var --- fs/move.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/move.py b/fs/move.py index bc287eb2..96a0bbe6 100644 --- a/fs/move.py +++ b/fs/move.py @@ -58,7 +58,7 @@ def move_file( resources (defaults to `False`). """ - if _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): + if src_fs.hassyspath(src_path) and dst_fs.hassyspath(dst_path): # if both filesystems have a syspath we create a new OSFS from a # common parent folder and use it to move the file. with manage_fs(src_fs) as _src_fs: From 86adaa8263a96345629abf3bc8f6630659d591d6 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 10:22:12 +0100 Subject: [PATCH 243/309] set "read_only": True in `WrapReadOnly.getmeta()` --- fs/wrap.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fs/wrap.py b/fs/wrap.py index 21fc10d1..ceb7b694 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -31,6 +31,7 @@ Dict, Iterator, IO, + Mapping, Optional, Text, Tuple, @@ -320,3 +321,11 @@ def touch(self, path): # type: (Text) -> None self.check() raise ResourceReadOnly(path) + + def getmeta(self, namespace="standard"): + # type: (Text) -> Mapping[Text, object] + self.check() + meta = self.delegate_fs().getmeta(namespace=namespace) + meta["read_only"] = True + meta["supports_rename"] = False + return meta From 014123b735ff3d35dece3b8a95426b645e53e17e Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 10:22:38 +0100 Subject: [PATCH 244/309] handle ro FSs in move_file optimization --- fs/move.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fs/move.py b/fs/move.py index 96a0bbe6..c3db533a 100644 --- a/fs/move.py +++ b/fs/move.py @@ -10,6 +10,7 @@ from . import open_fs from .copy import copy_dir from .copy import copy_file +from .errors import ResourceReadOnly from .opener import manage_fs from .path import frombase @@ -59,6 +60,12 @@ def move_file( """ if src_fs.hassyspath(src_path) and dst_fs.hassyspath(dst_path): + # we have to raise a ResourceReadOnly exception manually if a FS is read-only + if src_fs.getmeta().get("read_only", True): + raise ResourceReadOnly(src_fs, src_path) + if dst_fs.getmeta().get("read_only", True): + raise ResourceReadOnly(dst_fs, dst_path) + # if both filesystems have a syspath we create a new OSFS from a # common parent folder and use it to move the file. with manage_fs(src_fs) as _src_fs: From 14d4fa20255cb6148f72cb7707bac0d2ba03c82f Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 11:42:58 +0100 Subject: [PATCH 245/309] add test for two different MemFS instances --- tests/test_move.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_move.py b/tests/test_move.py index 2bae7340..c71ead9c 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -49,6 +49,13 @@ def test_move_file(self): self.assertEqual(a.readtext("here.txt"), "Content") self.assertFalse(b.exists("file.txt")) + def test_move_file_different_mems(self): + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("source.txt", "Source") + fs.move.move_file(src, "source.txt", dst, "dest.txt") + self.assertFalse(src.exists("source.txt")) + self.assertEqual(dst.readtext("dest.txt"), "Source") + def test_move_dir(self): namespaces = ("details", "modified") From a97c0c3334791032ea49d16f045ed9657a8a058f Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:00:07 +0100 Subject: [PATCH 246/309] support FS URLs --- fs/move.py | 60 +++++++++++++++++++++++----------------------- tests/test_move.py | 18 ++++++++++++++ 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/fs/move.py b/fs/move.py index c3db533a..7beca154 100644 --- a/fs/move.py +++ b/fs/move.py @@ -59,20 +59,28 @@ def move_file( resources (defaults to `False`). """ - if src_fs.hassyspath(src_path) and dst_fs.hassyspath(dst_path): - # we have to raise a ResourceReadOnly exception manually if a FS is read-only - if src_fs.getmeta().get("read_only", True): - raise ResourceReadOnly(src_fs, src_path) - if dst_fs.getmeta().get("read_only", True): - raise ResourceReadOnly(dst_fs, dst_path) - - # if both filesystems have a syspath we create a new OSFS from a - # common parent folder and use it to move the file. - with manage_fs(src_fs) as _src_fs: - with manage_fs(dst_fs, create=True) as _dst_fs: + with manage_fs(src_fs) as _src_fs: + with manage_fs(dst_fs, create=True) as _dst_fs: + if _src_fs is _dst_fs: + # Same filesystem, may be optimized + _src_fs.move( + src_path, dst_path, overwrite=True, preserve_time=preserve_time + ) + return + + if _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): + # if both filesystems have a syspath we create a new OSFS from a + # common parent folder and use it to move the file. + + # we have to raise ResourceReadOnly manually if a FS is read-only + if _src_fs.getmeta().get("read_only", True): + raise ResourceReadOnly(_src_fs, src_path) + if _dst_fs.getmeta().get("read_only", True): + raise ResourceReadOnly(_dst_fs, dst_path) + try: - src_syspath = src_fs.getsyspath(src_path) - dst_syspath = dst_fs.getsyspath(dst_path) + src_syspath = _src_fs.getsyspath(src_path) + dst_syspath = _dst_fs.getsyspath(dst_path) common = commonpath([src_syspath, dst_syspath]) rel_src = frombase(common, src_syspath) rel_dst = frombase(common, dst_syspath) @@ -84,24 +92,16 @@ def move_file( except ValueError: pass - with manage_fs(src_fs) as _src_fs: - with manage_fs(dst_fs, create=True) as _dst_fs: - if _src_fs is _dst_fs: - # Same filesystem, may be optimized - _src_fs.move( - src_path, dst_path, overwrite=True, preserve_time=preserve_time + # Standard copy and delete + with _src_fs.lock(), _dst_fs.lock(): + copy_file( + _src_fs, + src_path, + _dst_fs, + dst_path, + preserve_time=preserve_time, ) - else: - # Standard copy and delete - with _src_fs.lock(), _dst_fs.lock(): - copy_file( - _src_fs, - src_path, - _dst_fs, - dst_path, - preserve_time=preserve_time, - ) - _src_fs.remove(src_path) + _src_fs.remove(src_path) def move_dir( diff --git a/tests/test_move.py b/tests/test_move.py index c71ead9c..9bd2de5a 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -56,6 +56,24 @@ def test_move_file_different_mems(self): self.assertFalse(src.exists("source.txt")) self.assertEqual(dst.readtext("dest.txt"), "Source") + def test_move_file_fs_urls(self): + # create a temp dir to work on + with open_fs("temp://") as tmp: + path = tmp.getsyspath("/") + + tmp.writetext("file.txt", "Content") + tmp.makedir("subdir") + fs.move.move_file(path, "file.txt", join(path, "subdir"), "file.txt") + + self.assertFalse(tmp.exists("file.txt")) + self.assertEqual(tmp.readtext("subdir/file.txt"), "Content") + + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("source.txt", "Source") + fs.move.move_file(src, "source.txt", dst, "dest.txt") + self.assertFalse(src.exists("source.txt")) + self.assertEqual(dst.readtext("dest.txt"), "Source") + def test_move_dir(self): namespaces = ("details", "modified") From bbe4389c940f9bcd07d7ade92e2af7a930735566 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:19:41 +0100 Subject: [PATCH 247/309] clear use of fs url instead of str --- tests/test_move.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_move.py b/tests/test_move.py index 9bd2de5a..4630bf62 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -63,7 +63,12 @@ def test_move_file_fs_urls(self): tmp.writetext("file.txt", "Content") tmp.makedir("subdir") - fs.move.move_file(path, "file.txt", join(path, "subdir"), "file.txt") + fs.move.move_file( + "osfs://" + path, + "file.txt", + "osfs://" + join(path, "subdir"), + "file.txt", + ) self.assertFalse(tmp.exists("file.txt")) self.assertEqual(tmp.readtext("subdir/file.txt"), "Content") From 69022aa6bd3785a87ecd0e8008f7eb6787bbc2fe Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:19:52 +0100 Subject: [PATCH 248/309] add test for read-only sources --- tests/test_move.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_move.py b/tests/test_move.py index 4630bf62..b75c5796 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -6,7 +6,9 @@ import fs.move from fs import open_fs +from fs.errors import ResourceReadOnly from fs.path import join +from fs.wrap import read_only @parameterized_class(("preserve_time",), [(True,), (False,)]) @@ -79,6 +81,15 @@ def test_move_file_fs_urls(self): self.assertFalse(src.exists("source.txt")) self.assertEqual(dst.readtext("dest.txt"), "Source") + def test_move_file_read_only_source(self): + with open_fs("temp://") as tmp: + path = tmp.getsyspath("/") + tmp.writetext("file.txt", "Content") + src = read_only(open_fs(path)) + dst = tmp.makedir("sub") + with self.assertRaises(ResourceReadOnly): + fs.move.move_file(src, "file.txt", dst, "file.txt") + def test_move_dir(self): namespaces = ("details", "modified") From 545e016771e9d99c668371a5f844c299be381625 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:28:24 +0100 Subject: [PATCH 249/309] more tests for read only sources --- tests/test_move.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_move.py b/tests/test_move.py index b75c5796..fe59a0eb 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -89,6 +89,18 @@ def test_move_file_read_only_source(self): dst = tmp.makedir("sub") with self.assertRaises(ResourceReadOnly): fs.move.move_file(src, "file.txt", dst, "file.txt") + self.assertFalse(dst.exists("file.txt")) + self.assertTrue(src.exists("file.txt")) + + def test_move_file_read_only_mem_source(self): + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("file.txt", "Content") + sub = dst.makedir("sub") + src_ro = read_only(src) + with self.assertRaises(ResourceReadOnly): + fs.move.move_file(src_ro, "file.txt", sub, "file.txt") + self.assertFalse(sub.exists("file.txt")) + self.assertTrue(src.exists("file.txt")) def test_move_dir(self): namespaces = ("details", "modified") From fd577df13da06b0ee908aa8554016ae2e93cd432 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:37:17 +0100 Subject: [PATCH 250/309] clarify fallthrough and add writeable=True --- fs/move.py | 6 ++++-- tests/test_move.py | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/fs/move.py b/fs/move.py index 7beca154..728b69d1 100644 --- a/fs/move.py +++ b/fs/move.py @@ -59,8 +59,8 @@ def move_file( resources (defaults to `False`). """ - with manage_fs(src_fs) as _src_fs: - with manage_fs(dst_fs, create=True) as _dst_fs: + with manage_fs(src_fs, writeable=True) as _src_fs: + with manage_fs(dst_fs, writeable=True, create=True) as _dst_fs: if _src_fs is _dst_fs: # Same filesystem, may be optimized _src_fs.move( @@ -90,6 +90,8 @@ def move_file( base.move(rel_src, rel_dst, preserve_time=preserve_time) return # optimization worked, exit early except ValueError: + # This is raised if we cannot find a common base folder. + # In this case just fall through to the standard method. pass # Standard copy and delete diff --git a/tests/test_move.py b/tests/test_move.py index fe59a0eb..1e1668d0 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -89,7 +89,9 @@ def test_move_file_read_only_source(self): dst = tmp.makedir("sub") with self.assertRaises(ResourceReadOnly): fs.move.move_file(src, "file.txt", dst, "file.txt") - self.assertFalse(dst.exists("file.txt")) + self.assertFalse( + dst.exists("file.txt"), "file should not have been copied over" + ) self.assertTrue(src.exists("file.txt")) def test_move_file_read_only_mem_source(self): @@ -99,7 +101,9 @@ def test_move_file_read_only_mem_source(self): src_ro = read_only(src) with self.assertRaises(ResourceReadOnly): fs.move.move_file(src_ro, "file.txt", sub, "file.txt") - self.assertFalse(sub.exists("file.txt")) + self.assertFalse( + sub.exists("file.txt"), "file should not have been copied over" + ) self.assertTrue(src.exists("file.txt")) def test_move_dir(self): From 4edb1e092e17d7918440c4320f68d42f720b8456 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 12:47:08 +0100 Subject: [PATCH 251/309] cleanup destination if move fails --- fs/move.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/move.py b/fs/move.py index 728b69d1..b8178f9e 100644 --- a/fs/move.py +++ b/fs/move.py @@ -10,7 +10,7 @@ from . import open_fs from .copy import copy_dir from .copy import copy_file -from .errors import ResourceReadOnly +from .errors import FSError, ResourceReadOnly from .opener import manage_fs from .path import frombase @@ -103,7 +103,13 @@ def move_file( dst_path, preserve_time=preserve_time, ) - _src_fs.remove(src_path) + try: + _src_fs.remove(src_path) + except FSError as e: + # if the source cannot be removed we delete the copy on the + # destination + _dst_fs.remove(dst_path) + raise e def move_dir( From 20203d9159ecfc49f97d897d8ec86ccd9d92c91b Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 13:10:40 +0100 Subject: [PATCH 252/309] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249602dc..b0319992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). - Mark `fs.zipfs.ReadZipFS` as a case-sensitive filesystem ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). +- Optimized moving files between filesystems with syspaths. + ([#523](https://github.com/PyFilesystem/pyfilesystem2/pull/523)). +- Fixed `move.move_file` to clean up the copy on the destination in case of errors. ## [2.4.15] - 2022-02-07 From ad7a970bd4bfd4ac914a46a8acd9535c5e860112 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 13:51:06 +0100 Subject: [PATCH 253/309] raise exc in mange_fs if fs is not writeable --- CHANGELOG.md | 4 +++- fs/move.py | 11 ++--------- fs/opener/registry.py | 9 +++++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0319992..a453215e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). - Optimized moving files between filesystems with syspaths. ([#523](https://github.com/PyFilesystem/pyfilesystem2/pull/523)). -- Fixed `move.move_file` to clean up the copy on the destination in case of errors. +- Fixed `fs.move.move_file` to clean up the copy on the destination in case of errors. +- `fs.opener.manage_fs` with `writeable=True` will now raise a `ResourceReadOnly` + exception if the managed filesystem is not writeable. ## [2.4.15] - 2022-02-07 diff --git a/fs/move.py b/fs/move.py index b8178f9e..5f83cb92 100644 --- a/fs/move.py +++ b/fs/move.py @@ -71,13 +71,6 @@ def move_file( if _src_fs.hassyspath(src_path) and _dst_fs.hassyspath(dst_path): # if both filesystems have a syspath we create a new OSFS from a # common parent folder and use it to move the file. - - # we have to raise ResourceReadOnly manually if a FS is read-only - if _src_fs.getmeta().get("read_only", True): - raise ResourceReadOnly(_src_fs, src_path) - if _dst_fs.getmeta().get("read_only", True): - raise ResourceReadOnly(_dst_fs, dst_path) - try: src_syspath = _src_fs.getsyspath(src_path) dst_syspath = _dst_fs.getsyspath(dst_path) @@ -136,10 +129,10 @@ def move_dir( """ def src(): - return manage_fs(src_fs, writeable=False) + return manage_fs(src_fs, writeable=True) def dst(): - return manage_fs(dst_fs, create=True) + return manage_fs(dst_fs, writeable=True, create=True) with src() as _src_fs, dst() as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 54e2dda1..4621e64e 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -14,6 +14,7 @@ from .base import Opener from .errors import UnsupportedProtocol, EntryPointError +from ..errors import ResourceReadOnly from .parse import parse_fs_url if typing.TYPE_CHECKING: @@ -282,10 +283,18 @@ def manage_fs( """ from ..base import FS + def assert_writeable(fs): + if fs.getmeta().get("read_only", True): + raise ResourceReadOnly(path=str(fs_url)) + if isinstance(fs_url, FS): + if writeable: + assert_writeable(fs_url) yield fs_url else: _fs = self.open_fs(fs_url, create=create, writeable=writeable, cwd=cwd) + if writeable: + assert_writeable(_fs) try: yield _fs finally: From 1267194f785fa09738784fb4fa004db30e19f460 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 13:51:33 +0100 Subject: [PATCH 254/309] don't run tests twice in parameterized_class --- tests/test_move.py | 52 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 1e1668d0..96711741 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -12,7 +12,7 @@ @parameterized_class(("preserve_time",), [(True,), (False,)]) -class TestMove(unittest.TestCase): +class TestMovePreserveTime(unittest.TestCase): def test_move_fs(self): namespaces = ("details", "modified") @@ -40,6 +40,32 @@ def test_move_fs(self): self.assertEqual(dst_file1_info.modified, src_file1_info.modified) self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + def test_move_dir(self): + namespaces = ("details", "modified") + + src_fs = open_fs("mem://") + src_fs.makedirs("foo/bar") + src_fs.touch("test.txt") + src_fs.touch("foo/bar/baz.txt") + src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) + + dst_fs = open_fs("mem://") + dst_fs.create("test.txt") + dst_fs.setinfo("test.txt", {"details": {"modified": 1000000}}) + + fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time) + + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) + self.assertFalse(src_fs.exists("foo")) + self.assertTrue(src_fs.isfile("test.txt")) + + if self.preserve_time: + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + + +class TestMove(unittest.TestCase): def test_move_file(self): with open_fs("temp://") as temp: syspath = temp.getsyspath("/") @@ -106,26 +132,22 @@ def test_move_file_read_only_mem_source(self): ) self.assertTrue(src.exists("file.txt")) - def test_move_dir(self): - namespaces = ("details", "modified") - + def test_move_dir_cleanup(self): src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") - src_fs.touch("test.txt") src_fs.touch("foo/bar/baz.txt") - src_file2_info = src_fs.getinfo("foo/bar/baz.txt", namespaces) + src_fs.touch("foo/test.txt") dst_fs = open_fs("mem://") dst_fs.create("test.txt") - dst_fs.setinfo("test.txt", {"details": {"modified": 1000000}}) - fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time) + ro_src = read_only(src_fs) - self.assertTrue(dst_fs.isdir("bar")) - self.assertTrue(dst_fs.isfile("bar/baz.txt")) - self.assertFalse(src_fs.exists("foo")) - self.assertTrue(src_fs.isfile("test.txt")) + with self.assertRaises(ResourceReadOnly): + fs.move.move_dir(ro_src, "/foo", dst_fs, "/") - if self.preserve_time: - dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) - self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + self.assertTrue(src_fs.exists("foo/bar/baz.txt")) + self.assertTrue(src_fs.exists("foo/test.txt")) + self.assertFalse(dst_fs.isdir("bar")) + self.assertFalse(dst_fs.exists("bar/baz.txt")) + self.assertTrue(dst_fs.exists("test.txt")) From b0963bf0a2cad30becab4379e11f286b8e216f87 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 13:55:52 +0100 Subject: [PATCH 255/309] remove unneeded import --- fs/move.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/move.py b/fs/move.py index 5f83cb92..43779eb5 100644 --- a/fs/move.py +++ b/fs/move.py @@ -10,7 +10,7 @@ from . import open_fs from .copy import copy_dir from .copy import copy_file -from .errors import FSError, ResourceReadOnly +from .errors import FSError from .opener import manage_fs from .path import frombase From 4a102ce141eff2c537a79b093999a73d3842f5fd Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 14:07:23 +0100 Subject: [PATCH 256/309] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a453215e..5f581695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed `fs.move.move_file` to clean up the copy on the destination in case of errors. - `fs.opener.manage_fs` with `writeable=True` will now raise a `ResourceReadOnly` exception if the managed filesystem is not writeable. +- Mark `fs.wrap.WrapReadOnly` as read-only filesystem. ## [2.4.15] - 2022-02-07 From 763efc2fa7d78e4ad5149d98685bbe5ee38ddbd8 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 14:26:22 +0100 Subject: [PATCH 257/309] rename test case --- tests/test_move.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_move.py b/tests/test_move.py index 96711741..19256f1c 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -12,7 +12,7 @@ @parameterized_class(("preserve_time",), [(True,), (False,)]) -class TestMovePreserveTime(unittest.TestCase): +class TestMoveCheckTime(unittest.TestCase): def test_move_fs(self): namespaces = ("details", "modified") From e4904524cc3272f15ec5d96d28e543cbd9776343 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 14:39:23 +0100 Subject: [PATCH 258/309] formatting --- fs/move.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fs/move.py b/fs/move.py index 43779eb5..88fb871d 100644 --- a/fs/move.py +++ b/fs/move.py @@ -77,10 +77,9 @@ def move_file( common = commonpath([src_syspath, dst_syspath]) rel_src = frombase(common, src_syspath) rel_dst = frombase(common, dst_syspath) - with _src_fs.lock(), _dst_fs.lock(), open_fs( - common, writeable=True - ) as base: - base.move(rel_src, rel_dst, preserve_time=preserve_time) + with _src_fs.lock(), _dst_fs.lock(): + with open_fs(common, writeable=True) as base: + base.move(rel_src, rel_dst, preserve_time=preserve_time) return # optimization worked, exit early except ValueError: # This is raised if we cannot find a common base folder. From 0c02f53fbfbed97b0e5ff44a7bffa0c7c0967247 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 15:30:36 +0100 Subject: [PATCH 259/309] add python2.7 commonpath backport --- fs/_pathcompat.py | 40 ++++++++++++++++++++++++++++++++++++++++ fs/move.py | 13 ++++++------- 2 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 fs/_pathcompat.py diff --git a/fs/_pathcompat.py b/fs/_pathcompat.py new file mode 100644 index 00000000..9fa5689b --- /dev/null +++ b/fs/_pathcompat.py @@ -0,0 +1,40 @@ +try: + from os.path import commonpath +except ImportError: + # Return the longest common sub-path of the sequence of paths given as input. + # The paths are not normalized before comparing them (this is the + # responsibility of the caller). Any trailing separator is stripped from the + # returned path. + + def commonpath(paths): + """Given a sequence of path names, returns the longest common sub-path.""" + + if not paths: + raise ValueError("commonpath() arg is an empty sequence") + + paths = tuple(paths) + if isinstance(paths[0], bytes): + sep = b"/" + curdir = b"." + else: + sep = "/" + curdir = "." + + split_paths = [path.split(sep) for path in paths] + + try: + (isabs,) = set(p[:1] == sep for p in paths) + except ValueError: + raise ValueError("Can't mix absolute and relative paths") + + split_paths = [[c for c in s if c and c != curdir] for s in split_paths] + s1 = min(split_paths) + s2 = max(split_paths) + common = s1 + for i, c in enumerate(s1): + if c != s2[i]: + common = s1[:i] + break + + prefix = sep if isabs else sep[:0] + return prefix + sep.join(common) diff --git a/fs/move.py b/fs/move.py index 88fb871d..5f0e16f3 100644 --- a/fs/move.py +++ b/fs/move.py @@ -1,23 +1,22 @@ """Functions for moving files between filesystems. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -from os.path import commonpath import typing from . import open_fs -from .copy import copy_dir -from .copy import copy_file +from ._pathcompat import commonpath +from .copy import copy_dir, copy_file from .errors import FSError from .opener import manage_fs from .path import frombase if typing.TYPE_CHECKING: - from .base import FS from typing import Text, Union + from .base import FS + def move_fs( src_fs, # type: Union[Text, FS] @@ -80,7 +79,7 @@ def move_file( with _src_fs.lock(), _dst_fs.lock(): with open_fs(common, writeable=True) as base: base.move(rel_src, rel_dst, preserve_time=preserve_time) - return # optimization worked, exit early + return # optimization worked, exit early except ValueError: # This is raised if we cannot find a common base folder. # In this case just fall through to the standard method. From 9be038189d661d82c27623aa8b2265a7e1af3275 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 15:31:46 +0100 Subject: [PATCH 260/309] fix path in ResourceReadOnly exception --- fs/opener/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 4621e64e..9b955549 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -285,7 +285,7 @@ def manage_fs( def assert_writeable(fs): if fs.getmeta().get("read_only", True): - raise ResourceReadOnly(path=str(fs_url)) + raise ResourceReadOnly(path="/") if isinstance(fs_url, FS): if writeable: From baf70195999282d6c7a532363e9f13ebd703e21d Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 15:53:56 +0100 Subject: [PATCH 261/309] test_move cleanup --- tests/test_move.py | 83 ++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 19256f1c..432dc1e9 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -34,11 +34,37 @@ def test_move_fs(self): self.assertTrue(dst_fs.isfile("foo/bar/baz.txt")) self.assertTrue(src_fs.isempty("/")) - if self.preserve_time: - dst_file1_info = dst_fs.getinfo("test.txt", namespaces) - dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) - self.assertEqual(dst_file1_info.modified, src_file1_info.modified) - self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + dst_file1_info = dst_fs.getinfo("test.txt", namespaces) + dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) + self.assertEqual( + dst_file1_info.modified == src_file1_info.modified, + self.preserve_time, + ) + self.assertEqual( + dst_file2_info.modified == src_file2_info.modified, + self.preserve_time, + ) + + def test_move_file(self): + namespaces = ("details", "modified") + with open_fs("mem://") as src_fs, open_fs("mem://") as dst_fs: + src_fs.writetext("source.txt", "Source") + src_fs_file_info = src_fs.getinfo("source.txt", namespaces) + fs.move.move_file( + src_fs, + "source.txt", + dst_fs, + "dest.txt", + preserve_time=self.preserve_time, + ) + self.assertFalse(src_fs.exists("source.txt")) + self.assertEqual(dst_fs.readtext("dest.txt"), "Source") + + dst_fs_file_info = dst_fs.getinfo("dest.txt", namespaces) + self.assertEqual( + src_fs_file_info.modified == dst_fs_file_info.modified, + self.preserve_time, + ) def test_move_dir(self): namespaces = ("details", "modified") @@ -60,35 +86,26 @@ def test_move_dir(self): self.assertFalse(src_fs.exists("foo")) self.assertTrue(src_fs.isfile("test.txt")) - if self.preserve_time: - dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) - self.assertEqual(dst_file2_info.modified, src_file2_info.modified) + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual( + dst_file2_info.modified == src_file2_info.modified, self.preserve_time + ) class TestMove(unittest.TestCase): - def test_move_file(self): - with open_fs("temp://") as temp: - syspath = temp.getsyspath("/") - a = open_fs(syspath) - a.makedir("dir") - b = open_fs(join(syspath, "dir")) - b.writetext("file.txt", "Content") - fs.move.move_file(b, "file.txt", a, "here.txt") - self.assertEqual(a.readtext("here.txt"), "Content") - self.assertFalse(b.exists("file.txt")) - - def test_move_file_different_mems(self): - with open_fs("mem://") as src, open_fs("mem://") as dst: - src.writetext("source.txt", "Source") - fs.move.move_file(src, "source.txt", dst, "dest.txt") - self.assertFalse(src.exists("source.txt")) - self.assertEqual(dst.readtext("dest.txt"), "Source") + def test_move_file_tempfs(self): + with open_fs("temp://") as a, open_fs("temp://") as b: + dir_a = a.makedir("dir") + dir_b = b.makedir("subdir") + dir_b.writetext("file.txt", "Content") + fs.move.move_file(dir_b, "file.txt", dir_a, "here.txt") + self.assertEqual(a.readtext("dir/here.txt"), "Content") + self.assertFalse(b.exists("subdir/file.txt")) def test_move_file_fs_urls(self): # create a temp dir to work on with open_fs("temp://") as tmp: path = tmp.getsyspath("/") - tmp.writetext("file.txt", "Content") tmp.makedir("subdir") fs.move.move_file( @@ -97,7 +114,6 @@ def test_move_file_fs_urls(self): "osfs://" + join(path, "subdir"), "file.txt", ) - self.assertFalse(tmp.exists("file.txt")) self.assertEqual(tmp.readtext("subdir/file.txt"), "Content") @@ -107,7 +123,7 @@ def test_move_file_fs_urls(self): self.assertFalse(src.exists("source.txt")) self.assertEqual(dst.readtext("dest.txt"), "Source") - def test_move_file_read_only_source(self): + def test_move_file_same_fs_read_only_source(self): with open_fs("temp://") as tmp: path = tmp.getsyspath("/") tmp.writetext("file.txt", "Content") @@ -132,6 +148,17 @@ def test_move_file_read_only_mem_source(self): ) self.assertTrue(src.exists("file.txt")) + def test_move_file_read_only_mem_dest(self): + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("file.txt", "Content") + dst_ro = read_only(dst) + with self.assertRaises(ResourceReadOnly): + fs.move.move_file(src, "file.txt", dst_ro, "file.txt") + self.assertFalse( + dst_ro.exists("file.txt"), "file should not have been copied over" + ) + self.assertTrue(src.exists("file.txt")) + def test_move_dir_cleanup(self): src_fs = open_fs("mem://") src_fs.makedirs("foo/bar") From 69d1906e19ea7940ff23caf0a84302070278325c Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 15:58:32 +0100 Subject: [PATCH 262/309] check time on if self.preserve_time is set --- tests/test_move.py | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 432dc1e9..62abcde0 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -34,16 +34,11 @@ def test_move_fs(self): self.assertTrue(dst_fs.isfile("foo/bar/baz.txt")) self.assertTrue(src_fs.isempty("/")) - dst_file1_info = dst_fs.getinfo("test.txt", namespaces) - dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) - self.assertEqual( - dst_file1_info.modified == src_file1_info.modified, - self.preserve_time, - ) - self.assertEqual( - dst_file2_info.modified == src_file2_info.modified, - self.preserve_time, - ) + if self.preserve_time: + dst_file1_info = dst_fs.getinfo("test.txt", namespaces) + dst_file2_info = dst_fs.getinfo("foo/bar/baz.txt", namespaces) + self.assertEqual(dst_file1_info.modified, src_file1_info.modified) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) def test_move_file(self): namespaces = ("details", "modified") @@ -60,11 +55,9 @@ def test_move_file(self): self.assertFalse(src_fs.exists("source.txt")) self.assertEqual(dst_fs.readtext("dest.txt"), "Source") - dst_fs_file_info = dst_fs.getinfo("dest.txt", namespaces) - self.assertEqual( - src_fs_file_info.modified == dst_fs_file_info.modified, - self.preserve_time, - ) + if self.preserve_time: + dst_fs_file_info = dst_fs.getinfo("dest.txt", namespaces) + self.assertEqual(src_fs_file_info.modified, dst_fs_file_info.modified) def test_move_dir(self): namespaces = ("details", "modified") @@ -86,10 +79,9 @@ def test_move_dir(self): self.assertFalse(src_fs.exists("foo")) self.assertTrue(src_fs.isfile("test.txt")) - dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) - self.assertEqual( - dst_file2_info.modified == src_file2_info.modified, self.preserve_time - ) + if self.preserve_time: + dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) + self.assertEqual(dst_file2_info.modified, src_file2_info.modified) class TestMove(unittest.TestCase): From 7993af09276be6f83bc517542b0d5434998c7638 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 16:01:01 +0100 Subject: [PATCH 263/309] update changelog wording --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f581695..98660f83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Fixed `fs.move.move_file` to clean up the copy on the destination in case of errors. - `fs.opener.manage_fs` with `writeable=True` will now raise a `ResourceReadOnly` exception if the managed filesystem is not writeable. -- Mark `fs.wrap.WrapReadOnly` as read-only filesystem. +- Marked filesystems wrapped with `fs.wrap.WrapReadOnly` as read-only. ## [2.4.15] - 2022-02-07 From 9cfc5dd5e7297dbb1f859345c50b5b6e601f5ca3 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 16:26:30 +0100 Subject: [PATCH 264/309] revert import order --- fs/move.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fs/move.py b/fs/move.py index 5f0e16f3..b1db0a86 100644 --- a/fs/move.py +++ b/fs/move.py @@ -1,21 +1,22 @@ """Functions for moving files between filesystems. """ -from __future__ import print_function, unicode_literals +from __future__ import print_function +from __future__ import unicode_literals import typing from . import open_fs -from ._pathcompat import commonpath -from .copy import copy_dir, copy_file +from .copy import copy_dir +from .copy import copy_file from .errors import FSError from .opener import manage_fs from .path import frombase +from ._pathcompat import commonpath if typing.TYPE_CHECKING: - from typing import Text, Union - from .base import FS + from typing import Text, Union def move_fs( From 54d33747ec4d68ff45b5e71128a7dacc05bf119e Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 16:31:27 +0100 Subject: [PATCH 265/309] ignore flake8 C401 in _pathcompat --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index cd79fe78..1da24f30 100644 --- a/setup.cfg +++ b/setup.cfg @@ -96,6 +96,7 @@ per-file-ignores = tests/*:E501 fs/opener/*:F811 fs/_fscompat.py:F401 + fs/_pathcompat.py:C401 [isort] default_section = THIRD_PARTY From c051f23fc2638486138c1ef184ad1775348c573a Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 17:14:13 +0100 Subject: [PATCH 266/309] fix codestyle and typecheck errors --- fs/_pathcompat.py | 1 + fs/wrap.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/_pathcompat.py b/fs/_pathcompat.py index 9fa5689b..3d628662 100644 --- a/fs/_pathcompat.py +++ b/fs/_pathcompat.py @@ -1,3 +1,4 @@ +# mypy: ignore-errors try: from os.path import commonpath except ImportError: diff --git a/fs/wrap.py b/fs/wrap.py index ceb7b694..b037756f 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -325,7 +325,6 @@ def touch(self, path): def getmeta(self, namespace="standard"): # type: (Text) -> Mapping[Text, object] self.check() - meta = self.delegate_fs().getmeta(namespace=namespace) - meta["read_only"] = True - meta["supports_rename"] = False + meta = dict(self.delegate_fs().getmeta(namespace=namespace)) + meta.update(read_only=True, supports_rename=True) return meta From a54925039afcb7e2b7506bdca8654695c03d1a12 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 18:04:44 +0100 Subject: [PATCH 267/309] removed duplicated test code --- tests/test_move.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 62abcde0..d4ee7694 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -109,12 +109,6 @@ def test_move_file_fs_urls(self): self.assertFalse(tmp.exists("file.txt")) self.assertEqual(tmp.readtext("subdir/file.txt"), "Content") - with open_fs("mem://") as src, open_fs("mem://") as dst: - src.writetext("source.txt", "Source") - fs.move.move_file(src, "source.txt", dst, "dest.txt") - self.assertFalse(src.exists("source.txt")) - self.assertEqual(dst.readtext("dest.txt"), "Source") - def test_move_file_same_fs_read_only_source(self): with open_fs("temp://") as tmp: path = tmp.getsyspath("/") From 44cbe310b2410b2f7d6a741288c2f8c92c3ea54d Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 18:42:14 +0100 Subject: [PATCH 268/309] add `cleanup_dest_on_error` in `move_file` --- fs/move.py | 6 +++++- tests/test_move.py | 46 +++++++++++++++++++++++++++------------------- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/fs/move.py b/fs/move.py index b1db0a86..7ce3aca5 100644 --- a/fs/move.py +++ b/fs/move.py @@ -46,6 +46,7 @@ def move_file( dst_fs, # type: Union[Text, FS] dst_path, # type: Text preserve_time=False, # type: bool + cleanup_dest_on_error=True, # type: bool ): # type: (...) -> None """Move a file from one filesystem to another. @@ -57,6 +58,8 @@ def move_file( dst_path (str): Path to a file on ``dst_fs``. preserve_time (bool): If `True`, try to preserve mtime of the resources (defaults to `False`). + cleanup_dest_on_error (bool): If `True`, tries to delete the file copied to + dst_fs if deleting the file from src_fs fails. """ with manage_fs(src_fs, writeable=True) as _src_fs: @@ -100,7 +103,8 @@ def move_file( except FSError as e: # if the source cannot be removed we delete the copy on the # destination - _dst_fs.remove(dst_path) + if cleanup_dest_on_error: + _dst_fs.remove(dst_path) raise e diff --git a/tests/test_move.py b/tests/test_move.py index d4ee7694..ac15dd31 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -2,11 +2,17 @@ import unittest +try: + from unittest import mock +except ImportError: + import mock + from parameterized import parameterized_class + import fs.move from fs import open_fs -from fs.errors import ResourceReadOnly +from fs.errors import FSError, ResourceReadOnly from fs.path import join from fs.wrap import read_only @@ -145,22 +151,24 @@ def test_move_file_read_only_mem_dest(self): ) self.assertTrue(src.exists("file.txt")) - def test_move_dir_cleanup(self): - src_fs = open_fs("mem://") - src_fs.makedirs("foo/bar") - src_fs.touch("foo/bar/baz.txt") - src_fs.touch("foo/test.txt") - - dst_fs = open_fs("mem://") - dst_fs.create("test.txt") - - ro_src = read_only(src_fs) - - with self.assertRaises(ResourceReadOnly): - fs.move.move_dir(ro_src, "/foo", dst_fs, "/") + def test_move_file_cleanup_on_error(self): + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("file.txt", "Content") + with mock.patch.object(src, "remove") as mck: + mck.side_effect = FSError + with self.assertRaises(FSError): + fs.move.move_file(src, "file.txt", dst, "file.txt") + self.assertTrue(src.exists("file.txt")) + self.assertFalse(dst.exists("file.txt")) - self.assertTrue(src_fs.exists("foo/bar/baz.txt")) - self.assertTrue(src_fs.exists("foo/test.txt")) - self.assertFalse(dst_fs.isdir("bar")) - self.assertFalse(dst_fs.exists("bar/baz.txt")) - self.assertTrue(dst_fs.exists("test.txt")) + def test_move_file_no_cleanup_on_error(self): + with open_fs("mem://") as src, open_fs("mem://") as dst: + src.writetext("file.txt", "Content") + with mock.patch.object(src, "remove") as mck: + mck.side_effect = FSError + with self.assertRaises(FSError): + fs.move.move_file( + src, "file.txt", dst, "file.txt", cleanup_dest_on_error=False + ) + self.assertTrue(src.exists("file.txt")) + self.assertTrue(dst.exists("file.txt")) From 696c4cabd851c7044b7e69f0e3cd9cbebf639596 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 25 Mar 2022 18:45:10 +0100 Subject: [PATCH 269/309] use non-overlapping osfs urls --- tests/test_move.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index ac15dd31..57685fe2 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -104,16 +104,17 @@ def test_move_file_fs_urls(self): # create a temp dir to work on with open_fs("temp://") as tmp: path = tmp.getsyspath("/") - tmp.writetext("file.txt", "Content") - tmp.makedir("subdir") + subdir_src = tmp.makedir("subdir_src") + subdir_src.writetext("file.txt", "Content") + tmp.makedir("subdir_dst") fs.move.move_file( - "osfs://" + path, + "osfs://" + join(path, "subdir_src"), "file.txt", - "osfs://" + join(path, "subdir"), + "osfs://" + join(path, "subdir_dst"), "file.txt", ) - self.assertFalse(tmp.exists("file.txt")) - self.assertEqual(tmp.readtext("subdir/file.txt"), "Content") + self.assertFalse(subdir_src.exists("file.txt")) + self.assertEqual(tmp.readtext("subdir_dst/file.txt"), "Content") def test_move_file_same_fs_read_only_source(self): with open_fs("temp://") as tmp: From 891d1fc8df1dfe7fed4a037c5fd82ca7058a3ff2 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 28 Mar 2022 10:33:41 +0200 Subject: [PATCH 270/309] use OSFS instead of fs_open, rename cleanup_dst_on_error param --- fs/move.py | 55 +++++++++++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/fs/move.py b/fs/move.py index 7ce3aca5..c345e2cc 100644 --- a/fs/move.py +++ b/fs/move.py @@ -6,11 +6,11 @@ import typing -from . import open_fs from .copy import copy_dir from .copy import copy_file from .errors import FSError from .opener import manage_fs +from .osfs import OSFS from .path import frombase from ._pathcompat import commonpath @@ -46,7 +46,7 @@ def move_file( dst_fs, # type: Union[Text, FS] dst_path, # type: Text preserve_time=False, # type: bool - cleanup_dest_on_error=True, # type: bool + cleanup_dst_on_error=True, # type: bool ): # type: (...) -> None """Move a file from one filesystem to another. @@ -58,8 +58,8 @@ def move_file( dst_path (str): Path to a file on ``dst_fs``. preserve_time (bool): If `True`, try to preserve mtime of the resources (defaults to `False`). - cleanup_dest_on_error (bool): If `True`, tries to delete the file copied to - dst_fs if deleting the file from src_fs fails. + cleanup_dst_on_error (bool): If `True`, tries to delete the file copied to + `dst_fs` if deleting the file from `src_fs` fails (defaults to `True`). """ with manage_fs(src_fs, writeable=True) as _src_fs: @@ -78,12 +78,13 @@ def move_file( src_syspath = _src_fs.getsyspath(src_path) dst_syspath = _dst_fs.getsyspath(dst_path) common = commonpath([src_syspath, dst_syspath]) - rel_src = frombase(common, src_syspath) - rel_dst = frombase(common, dst_syspath) - with _src_fs.lock(), _dst_fs.lock(): - with open_fs(common, writeable=True) as base: - base.move(rel_src, rel_dst, preserve_time=preserve_time) - return # optimization worked, exit early + if common: + rel_src = frombase(common, src_syspath) + rel_dst = frombase(common, dst_syspath) + with _src_fs.lock(), _dst_fs.lock(): + with OSFS(common) as base: + base.move(rel_src, rel_dst, preserve_time=preserve_time) + return # optimization worked, exit early except ValueError: # This is raised if we cannot find a common base folder. # In this case just fall through to the standard method. @@ -103,7 +104,7 @@ def move_file( except FSError as e: # if the source cannot be removed we delete the copy on the # destination - if cleanup_dest_on_error: + if cleanup_dst_on_error: _dst_fs.remove(dst_path) raise e @@ -130,22 +131,16 @@ def move_dir( resources (defaults to `False`). """ - - def src(): - return manage_fs(src_fs, writeable=True) - - def dst(): - return manage_fs(dst_fs, writeable=True, create=True) - - with src() as _src_fs, dst() as _dst_fs: - with _src_fs.lock(), _dst_fs.lock(): - _dst_fs.makedir(dst_path, recreate=True) - copy_dir( - src_fs, - src_path, - dst_fs, - dst_path, - workers=workers, - preserve_time=preserve_time, - ) - _src_fs.removetree(src_path) + with manage_fs(src_fs, writeable=True) as _src_fs: + with manage_fs(dst_fs, writeable=True, create=True) as _dst_fs: + with _src_fs.lock(), _dst_fs.lock(): + _dst_fs.makedir(dst_path, recreate=True) + copy_dir( + src_fs, + src_path, + dst_fs, + dst_path, + workers=workers, + preserve_time=preserve_time, + ) + _src_fs.removetree(src_path) From 4d80b2272430145fc40afed15ba1ed66ffeb31c2 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 28 Mar 2022 10:34:06 +0200 Subject: [PATCH 271/309] fix `supports_rename` in `WrapReadOnly` --- fs/wrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/wrap.py b/fs/wrap.py index b037756f..113ca28f 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -326,5 +326,5 @@ def getmeta(self, namespace="standard"): # type: (Text) -> Mapping[Text, object] self.check() meta = dict(self.delegate_fs().getmeta(namespace=namespace)) - meta.update(read_only=True, supports_rename=True) + meta.update(read_only=True, supports_rename=False) return meta From 78f669605a3b216a49a81cb8c94e82a4c99b5b43 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 28 Mar 2022 10:34:15 +0200 Subject: [PATCH 272/309] cleanup test_move filenames --- tests/test_move.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 57685fe2..d586a88e 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -92,13 +92,13 @@ def test_move_dir(self): class TestMove(unittest.TestCase): def test_move_file_tempfs(self): - with open_fs("temp://") as a, open_fs("temp://") as b: - dir_a = a.makedir("dir") - dir_b = b.makedir("subdir") - dir_b.writetext("file.txt", "Content") - fs.move.move_file(dir_b, "file.txt", dir_a, "here.txt") - self.assertEqual(a.readtext("dir/here.txt"), "Content") - self.assertFalse(b.exists("subdir/file.txt")) + with open_fs("temp://") as src, open_fs("temp://") as dst: + src_dir = src.makedir("Some subfolder") + src_dir.writetext("file.txt", "Content") + dst_dir = dst.makedir("dest dir") + fs.move.move_file(src_dir, "file.txt", dst_dir, "target.txt") + self.assertEqual(dst.readtext("dest dir/target.txt"), "Content") + self.assertFalse(src.exists("Some subfolder/file.txt")) def test_move_file_fs_urls(self): # create a temp dir to work on @@ -111,10 +111,10 @@ def test_move_file_fs_urls(self): "osfs://" + join(path, "subdir_src"), "file.txt", "osfs://" + join(path, "subdir_dst"), - "file.txt", + "target.txt", ) self.assertFalse(subdir_src.exists("file.txt")) - self.assertEqual(tmp.readtext("subdir_dst/file.txt"), "Content") + self.assertEqual(tmp.readtext("subdir_dst/target.txt"), "Content") def test_move_file_same_fs_read_only_source(self): with open_fs("temp://") as tmp: @@ -123,21 +123,21 @@ def test_move_file_same_fs_read_only_source(self): src = read_only(open_fs(path)) dst = tmp.makedir("sub") with self.assertRaises(ResourceReadOnly): - fs.move.move_file(src, "file.txt", dst, "file.txt") + fs.move.move_file(src, "file.txt", dst, "target_file.txt") self.assertFalse( - dst.exists("file.txt"), "file should not have been copied over" + dst.exists("target_file.txt"), "file should not have been copied over" ) self.assertTrue(src.exists("file.txt")) def test_move_file_read_only_mem_source(self): with open_fs("mem://") as src, open_fs("mem://") as dst: src.writetext("file.txt", "Content") - sub = dst.makedir("sub") + dst_sub = dst.makedir("sub") src_ro = read_only(src) with self.assertRaises(ResourceReadOnly): - fs.move.move_file(src_ro, "file.txt", sub, "file.txt") + fs.move.move_file(src_ro, "file.txt", dst_sub, "target.txt") self.assertFalse( - sub.exists("file.txt"), "file should not have been copied over" + dst_sub.exists("target.txt"), "file should not have been copied over" ) self.assertTrue(src.exists("file.txt")) @@ -146,9 +146,9 @@ def test_move_file_read_only_mem_dest(self): src.writetext("file.txt", "Content") dst_ro = read_only(dst) with self.assertRaises(ResourceReadOnly): - fs.move.move_file(src, "file.txt", dst_ro, "file.txt") + fs.move.move_file(src, "file.txt", dst_ro, "target.txt") self.assertFalse( - dst_ro.exists("file.txt"), "file should not have been copied over" + dst_ro.exists("target.txt"), "file should not have been copied over" ) self.assertTrue(src.exists("file.txt")) @@ -158,9 +158,9 @@ def test_move_file_cleanup_on_error(self): with mock.patch.object(src, "remove") as mck: mck.side_effect = FSError with self.assertRaises(FSError): - fs.move.move_file(src, "file.txt", dst, "file.txt") + fs.move.move_file(src, "file.txt", dst, "target.txt") self.assertTrue(src.exists("file.txt")) - self.assertFalse(dst.exists("file.txt")) + self.assertFalse(dst.exists("target.txt")) def test_move_file_no_cleanup_on_error(self): with open_fs("mem://") as src, open_fs("mem://") as dst: @@ -169,7 +169,7 @@ def test_move_file_no_cleanup_on_error(self): mck.side_effect = FSError with self.assertRaises(FSError): fs.move.move_file( - src, "file.txt", dst, "file.txt", cleanup_dest_on_error=False + src, "file.txt", dst, "target.txt", cleanup_dst_on_error=False ) self.assertTrue(src.exists("file.txt")) - self.assertTrue(dst.exists("file.txt")) + self.assertTrue(dst.exists("target.txt")) From 46fbc7163dc21058225836acf93c7994d2200903 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Wed, 30 Mar 2022 14:07:09 +0200 Subject: [PATCH 273/309] Fix circular import between `fs.base`, `fs.osfs` and `fs.move` --- fs/base.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/base.py b/fs/base.py index 83c36218..91152a60 100644 --- a/fs/base.py +++ b/fs/base.py @@ -21,7 +21,7 @@ import six -from . import copy, errors, fsencode, iotools, move, tools, walk, wildcard +from . import copy, errors, fsencode, iotools, tools, walk, wildcard from .copy import copy_modified_time from .glob import BoundGlobber from .mode import validate_open_mode @@ -1083,10 +1083,12 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): ancestors is not a directory. """ + from .move import move_dir + with self._lock: if not create and not self.exists(dst_path): raise errors.ResourceNotFound(dst_path) - move.move_dir(self, src_path, self, dst_path, preserve_time=preserve_time) + move_dir(self, src_path, self, dst_path, preserve_time=preserve_time) def makedirs( self, From 5675f84cb3888092f407cfbc51224f639dd7caa5 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 1 Apr 2022 10:59:54 +0200 Subject: [PATCH 274/309] cleanup and docs --- fs/move.py | 2 +- tests/test_move.py | 39 +++++++++++++++++---------------------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/fs/move.py b/fs/move.py index c345e2cc..5b200963 100644 --- a/fs/move.py +++ b/fs/move.py @@ -59,7 +59,7 @@ def move_file( preserve_time (bool): If `True`, try to preserve mtime of the resources (defaults to `False`). cleanup_dst_on_error (bool): If `True`, tries to delete the file copied to - `dst_fs` if deleting the file from `src_fs` fails (defaults to `True`). + ``dst_fs`` if deleting the file from ``src_fs`` fails (defaults to `True`). """ with manage_fs(src_fs, writeable=True) as _src_fs: diff --git a/tests/test_move.py b/tests/test_move.py index d586a88e..4f3a94de 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -7,7 +7,7 @@ except ImportError: import mock -from parameterized import parameterized_class +from parameterized import parameterized, parameterized_class import fs.move @@ -80,10 +80,10 @@ def test_move_dir(self): fs.move.move_dir(src_fs, "/foo", dst_fs, "/", preserve_time=self.preserve_time) - self.assertTrue(dst_fs.isdir("bar")) - self.assertTrue(dst_fs.isfile("bar/baz.txt")) self.assertFalse(src_fs.exists("foo")) self.assertTrue(src_fs.isfile("test.txt")) + self.assertTrue(dst_fs.isdir("bar")) + self.assertTrue(dst_fs.isfile("bar/baz.txt")) if self.preserve_time: dst_file2_info = dst_fs.getinfo("bar/baz.txt", namespaces) @@ -104,8 +104,8 @@ def test_move_file_fs_urls(self): # create a temp dir to work on with open_fs("temp://") as tmp: path = tmp.getsyspath("/") - subdir_src = tmp.makedir("subdir_src") - subdir_src.writetext("file.txt", "Content") + tmp.makedir("subdir_src") + tmp.writetext("subdir_src/file.txt", "Content") tmp.makedir("subdir_dst") fs.move.move_file( "osfs://" + join(path, "subdir_src"), @@ -113,7 +113,7 @@ def test_move_file_fs_urls(self): "osfs://" + join(path, "subdir_dst"), "target.txt", ) - self.assertFalse(subdir_src.exists("file.txt")) + self.assertFalse(tmp.exists("subdir_src/file.txt")) self.assertEqual(tmp.readtext("subdir_dst/target.txt"), "Content") def test_move_file_same_fs_read_only_source(self): @@ -124,10 +124,10 @@ def test_move_file_same_fs_read_only_source(self): dst = tmp.makedir("sub") with self.assertRaises(ResourceReadOnly): fs.move.move_file(src, "file.txt", dst, "target_file.txt") + self.assertTrue(src.exists("file.txt")) self.assertFalse( dst.exists("target_file.txt"), "file should not have been copied over" ) - self.assertTrue(src.exists("file.txt")) def test_move_file_read_only_mem_source(self): with open_fs("mem://") as src, open_fs("mem://") as dst: @@ -136,10 +136,10 @@ def test_move_file_read_only_mem_source(self): src_ro = read_only(src) with self.assertRaises(ResourceReadOnly): fs.move.move_file(src_ro, "file.txt", dst_sub, "target.txt") + self.assertTrue(src.exists("file.txt")) self.assertFalse( dst_sub.exists("target.txt"), "file should not have been copied over" ) - self.assertTrue(src.exists("file.txt")) def test_move_file_read_only_mem_dest(self): with open_fs("mem://") as src, open_fs("mem://") as dst: @@ -147,29 +147,24 @@ def test_move_file_read_only_mem_dest(self): dst_ro = read_only(dst) with self.assertRaises(ResourceReadOnly): fs.move.move_file(src, "file.txt", dst_ro, "target.txt") + self.assertTrue(src.exists("file.txt")) self.assertFalse( dst_ro.exists("target.txt"), "file should not have been copied over" ) - self.assertTrue(src.exists("file.txt")) - - def test_move_file_cleanup_on_error(self): - with open_fs("mem://") as src, open_fs("mem://") as dst: - src.writetext("file.txt", "Content") - with mock.patch.object(src, "remove") as mck: - mck.side_effect = FSError - with self.assertRaises(FSError): - fs.move.move_file(src, "file.txt", dst, "target.txt") - self.assertTrue(src.exists("file.txt")) - self.assertFalse(dst.exists("target.txt")) - def test_move_file_no_cleanup_on_error(self): + @parameterized.expand([(True,), (False,)]) + def test_move_file_cleanup_on_error(self, cleanup): with open_fs("mem://") as src, open_fs("mem://") as dst: src.writetext("file.txt", "Content") with mock.patch.object(src, "remove") as mck: mck.side_effect = FSError with self.assertRaises(FSError): fs.move.move_file( - src, "file.txt", dst, "target.txt", cleanup_dst_on_error=False + src, + "file.txt", + dst, + "target.txt", + cleanup_dst_on_error=cleanup, ) self.assertTrue(src.exists("file.txt")) - self.assertTrue(dst.exists("target.txt")) + self.assertEqual(not dst.exists("target.txt"), cleanup) From da33922a48840f67a0f8b833c38ae68e65faa69f Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 1 Apr 2022 11:06:58 +0200 Subject: [PATCH 275/309] test `src` first, then `dst` --- tests/test_move.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_move.py b/tests/test_move.py index 4f3a94de..2ec1b71b 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -35,10 +35,10 @@ def test_move_fs(self): fs.move.move_fs(src_fs, dst_fs, preserve_time=self.preserve_time) + self.assertTrue(src_fs.isempty("/")) self.assertTrue(dst_fs.isdir("foo/bar")) self.assertTrue(dst_fs.isfile("test.txt")) self.assertTrue(dst_fs.isfile("foo/bar/baz.txt")) - self.assertTrue(src_fs.isempty("/")) if self.preserve_time: dst_file1_info = dst_fs.getinfo("test.txt", namespaces) @@ -97,8 +97,8 @@ def test_move_file_tempfs(self): src_dir.writetext("file.txt", "Content") dst_dir = dst.makedir("dest dir") fs.move.move_file(src_dir, "file.txt", dst_dir, "target.txt") - self.assertEqual(dst.readtext("dest dir/target.txt"), "Content") self.assertFalse(src.exists("Some subfolder/file.txt")) + self.assertEqual(dst.readtext("dest dir/target.txt"), "Content") def test_move_file_fs_urls(self): # create a temp dir to work on From 2b3b0b05cd726f76b4d111ab691036f78bc5d240 Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 2 May 2022 10:09:23 +0200 Subject: [PATCH 276/309] Revert switch from `appdirs` to `platformdirs` for backward compatibility --- CHANGELOG.md | 2 -- fs/appfs.py | 6 +++--- setup.cfg | 2 +- tests/test_appfs.py | 2 +- tests/test_opener.py | 6 +++--- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98660f83..c9325628 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Changed -- Replaced `appdirs` with `platformdirs` dependency - ([#489](https://github.com/PyFilesystem/pyfilesystem2/pull/489)). - Make `fs.zipfs._ZipExtFile` use the seeking mechanism implemented in the Python standard library in Python version 3.7 and later ([#527](https://github.com/PyFilesystem/pyfilesystem2/pull/527)). diff --git a/fs/appfs.py b/fs/appfs.py index 4c337e44..131ea8a8 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -5,7 +5,7 @@ subclasses of `~fs.osfs.OSFS`. """ -# Thanks to authors of https://pypi.org/project/platformdirs +# Thanks to authors of https://pypi.org/project/appdirs # see http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx @@ -16,7 +16,7 @@ from .osfs import OSFS from ._repr import make_repr -from platformdirs import PlatformDirs +from appdirs import AppDirs if typing.TYPE_CHECKING: from typing import Optional, Text @@ -78,7 +78,7 @@ def __init__( will be created if it does not exist. """ - self.app_dirs = PlatformDirs(appname, author, version, roaming) + self.app_dirs = AppDirs(appname, author, version, roaming) self._create = create super(_AppFS, self).__init__( getattr(self.app_dirs, self.app_dir), create=create diff --git a/setup.cfg b/setup.cfg index 1da24f30..d67bb927 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ packages = find: setup_requires = setuptools >=38.3.0 install_requires = - platformdirs~=2.0.2 + appdirs~=1.4.3 setuptools six ~=1.10 enum34 ~=1.1.6 ; python_version < '3.4' diff --git a/tests/test_appfs.py b/tests/test_appfs.py index 8cf4b9eb..acc8a7f7 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -30,7 +30,7 @@ def tearDownClass(cls): def make_fs(self): with mock.patch( - "platformdirs.{}".format(self.AppFS.app_dir), + "appdirs.{}".format(self.AppFS.app_dir), autospec=True, spec_set=True, return_value=tempfile.mkdtemp(dir=self.tmpdir), diff --git a/tests/test_opener.py b/tests/test_opener.py index 49b1a50d..bc2f5cd7 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -264,7 +264,7 @@ def test_open_fs(self): mem_fs_2 = opener.open_fs(mem_fs) self.assertEqual(mem_fs, mem_fs_2) - @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_open_userdata(self, app_dir): app_dir.return_value = self.tmpdir @@ -276,7 +276,7 @@ def test_open_userdata(self, app_dir): self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, "1.0") - @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_open_userdata_no_version(self, app_dir): app_dir.return_value = self.tmpdir @@ -285,7 +285,7 @@ def test_open_userdata_no_version(self, app_dir): self.assertEqual(app_fs.app_dirs.appauthor, "willmcgugan") self.assertEqual(app_fs.app_dirs.version, None) - @mock.patch("platformdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) + @mock.patch("appdirs.{}".format(UserDataFS.app_dir), autospec=True, spec_set=True) def test_user_data_opener(self, app_dir): app_dir.return_value = self.tmpdir From 486cf3aa49532fd70d22fb19429cc6d7c2eceb0a Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 2 May 2022 10:10:42 +0200 Subject: [PATCH 277/309] Fix invalid `isort` configuration in `setup.cfg` --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d67bb927..154f7312 100644 --- a/setup.cfg +++ b/setup.cfg @@ -99,10 +99,12 @@ per-file-ignores = fs/_pathcompat.py:C401 [isort] -default_section = THIRD_PARTY +default_section = THIRDPARTY known_first_party = fs known_standard_library = typing line_length = 88 +profile = black +skip_gitignore = true # --- Test and coverage configuration ------------------------------------------ From 75cf00aaa36d1507ca956c2d06700c93e02acfcd Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 2 May 2022 10:12:38 +0200 Subject: [PATCH 278/309] Fix import order with new `isort` options --- docs/source/conf.py | 4 ++-- examples/count_py.py | 1 - examples/find_dups.py | 4 ++-- examples/rm_pyc.py | 1 - examples/upload.py | 3 ++- fs/__init__.py | 4 ++-- fs/_bulk.py | 8 +++++--- fs/_fscompat.py | 4 ++-- fs/_ftp_parse.py | 7 ++----- fs/_typing.py | 2 +- fs/_tzcompat.py | 3 +-- fs/_url_tools.py | 5 +++-- fs/appfs.py | 6 +++--- fs/base.py | 18 ++++++++++-------- fs/compress.py | 15 +++++++-------- fs/constants.py | 1 - fs/copy.py | 2 ++ fs/enums.py | 3 +-- fs/error_tools.py | 12 ++++++------ fs/errors.py | 5 ++--- fs/filesize.py | 5 ++--- fs/ftpfs.py | 34 ++++++++++++++-------------------- fs/glob.py | 11 ++++++----- fs/info.py | 13 ++++++------- fs/iotools.py | 20 ++++++-------------- fs/lrucache.py | 5 ++--- fs/memoryfs.py | 23 +++++++++++------------ fs/mirror.py | 4 ++-- fs/mode.py | 4 +--- fs/mountfs.py | 14 +++++--------- fs/move.py | 11 +++++------ fs/multifs.py | 13 ++++++------- fs/opener/__init__.py | 6 +++--- fs/opener/appfs.py | 11 +++++------ fs/opener/base.py | 3 ++- fs/opener/ftpfs.py | 7 +++---- fs/opener/memoryfs.py | 7 +++---- fs/opener/osfs.py | 10 +++++----- fs/opener/parse.py | 8 +++----- fs/opener/registry.py | 24 +++++++----------------- fs/opener/tarfs.py | 9 ++++----- fs/opener/tempfs.py | 7 +++---- fs/opener/zipfs.py | 9 ++++----- fs/osfs.py | 27 +++++++++++++-------------- fs/path.py | 6 +++--- fs/permissions.py | 4 +--- fs/subfs.py | 8 ++++---- fs/tarfs.py | 20 ++++++++++---------- fs/tempfs.py | 7 +++---- fs/test.py | 15 +++++---------- fs/time.py | 4 ++-- fs/tools.py | 12 ++++-------- fs/tree.py | 4 ++-- fs/walk.py | 12 +++++------- fs/wildcard.py | 7 ++++--- fs/wrap.py | 17 +++++++++-------- fs/wrapfs.py | 14 ++++++++------ fs/zipfs.py | 12 ++++++------ setup.cfg | 2 +- setup.py | 1 + tests/test_appfs.py | 3 +-- tests/test_archives.py | 6 ++---- tests/test_base.py | 2 +- tests/test_copy.py | 9 ++++----- tests/test_doctest.py | 4 ++-- tests/test_encoding.py | 4 +--- tests/test_enums.py | 3 +-- tests/test_errors.py | 1 - tests/test_filesize.py | 4 ++-- tests/test_fscompat.py | 5 ++--- tests/test_ftpfs.py | 15 +++++---------- tests/test_glob.py | 3 +-- tests/test_imports.py | 1 + tests/test_info.py | 2 +- tests/test_iotools.py | 7 ++----- tests/test_memoryfs.py | 3 +-- tests/test_mirror.py | 3 +-- tests/test_mode.py | 3 +-- tests/test_mountfs.py | 2 +- tests/test_move.py | 1 - tests/test_multifs.py | 5 ++--- tests/test_new_name.py | 1 - tests/test_opener.py | 11 ++++++----- tests/test_osfs.py | 11 +++++------ tests/test_path.py | 2 +- tests/test_permissions.py | 6 ++---- tests/test_subfs.py | 3 ++- tests/test_tarfs.py | 4 ++-- tests/test_tempfs.py | 2 +- tests/test_time.py | 4 ++-- tests/test_tools.py | 3 +-- tests/test_tree.py | 3 +-- tests/test_walk.py | 4 ++-- tests/test_wrap.py | 2 +- tests/test_wrapfs.py | 1 - tests/test_zipfs.py | 10 +++++----- 96 files changed, 305 insertions(+), 386 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2ec980f2..661cd0c5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,9 +13,8 @@ # serve to show the default. import sys -import os - +import os import sphinx_rtd_theme html_theme = "sphinx_rtd_theme" @@ -72,6 +71,7 @@ # built documents. # from fs import __version__ + # The short X.Y version. version = '.'.join(__version__.split('.')[:2]) # The full version, including alpha/beta/rc tags. diff --git a/examples/count_py.py b/examples/count_py.py index 1f38d1e2..1a6dd670 100644 --- a/examples/count_py.py +++ b/examples/count_py.py @@ -11,7 +11,6 @@ from fs import open_fs from fs.filesize import traditional - fs_url = sys.argv[1] count = 0 diff --git a/examples/find_dups.py b/examples/find_dups.py index adc8b2cc..269509f3 100644 --- a/examples/find_dups.py +++ b/examples/find_dups.py @@ -7,11 +7,11 @@ """ -from collections import defaultdict import sys -from fs import open_fs +from collections import defaultdict +from fs import open_fs hashes = defaultdict(list) with open_fs(sys.argv[1]) as fs: diff --git a/examples/rm_pyc.py b/examples/rm_pyc.py index 71f46b17..9d95f5d7 100644 --- a/examples/rm_pyc.py +++ b/examples/rm_pyc.py @@ -11,7 +11,6 @@ from fs import open_fs - with open_fs(sys.argv[1]) as fs: count = fs.glob("**/*.pyc").remove() print(f"{count} .pyc files remove") diff --git a/examples/upload.py b/examples/upload.py index 04a0e152..77e5d401 100644 --- a/examples/upload.py +++ b/examples/upload.py @@ -12,9 +12,10 @@ """ -import os import sys +import os + from fs import open_fs _, file_path, fs_url = sys.argv diff --git a/fs/__init__.py b/fs/__init__.py index 885141e3..97dc55ba 100644 --- a/fs/__init__.py +++ b/fs/__init__.py @@ -3,10 +3,10 @@ __import__("pkg_resources").declare_namespace(__name__) # type: ignore +from . import path +from ._fscompat import fsdecode, fsencode from ._version import __version__ from .enums import ResourceType, Seek from .opener import open_fs -from ._fscompat import fsencode, fsdecode -from . import path __all__ = ["__version__", "ResourceType", "Seek", "open_fs"] diff --git a/fs/_bulk.py b/fs/_bulk.py index 12eef85e..caba0c58 100644 --- a/fs/_bulk.py +++ b/fs/_bulk.py @@ -6,9 +6,9 @@ from __future__ import unicode_literals -import threading import typing +import threading from six.moves.queue import Queue from .copy import copy_file_internal, copy_modified_time @@ -16,9 +16,11 @@ from .tools import copy_file_data if typing.TYPE_CHECKING: - from .base import FS + from typing import IO, List, Optional, Text, Tuple, Type + from types import TracebackType - from typing import List, Optional, Text, Type, IO, Tuple + + from .base import FS class _Worker(threading.Thread): diff --git a/fs/_fscompat.py b/fs/_fscompat.py index de59fa29..fa7d2c0b 100644 --- a/fs/_fscompat.py +++ b/fs/_fscompat.py @@ -1,9 +1,9 @@ import six try: - from os import fsencode, fsdecode + from os import fsdecode, fsencode except ImportError: - from backports.os import fsencode, fsdecode # type: ignore + from backports.os import fsdecode, fsencode # type: ignore try: from os import fspath diff --git a/fs/_ftp_parse.py b/fs/_ftp_parse.py index 42c0720e..16c581eb 100644 --- a/fs/_ftp_parse.py +++ b/fs/_ftp_parse.py @@ -1,10 +1,8 @@ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals -import unicodedata import re import time +import unicodedata from datetime import datetime try: @@ -15,7 +13,6 @@ from .enums import ResourceType from .permissions import Permissions - EPOCH_DT = datetime.fromtimestamp(0, timezone.utc) diff --git a/fs/_typing.py b/fs/_typing.py index 7c1f2275..0c80b8ef 100644 --- a/fs/_typing.py +++ b/fs/_typing.py @@ -3,8 +3,8 @@ """ import sys -import six +import six _PY = sys.version_info diff --git a/fs/_tzcompat.py b/fs/_tzcompat.py index 282859b2..135a7b32 100644 --- a/fs/_tzcompat.py +++ b/fs/_tzcompat.py @@ -4,8 +4,7 @@ https://docs.python.org/2.7/library/datetime.html#tzinfo-objects """ -from datetime import tzinfo, timedelta - +from datetime import timedelta, tzinfo ZERO = timedelta(0) diff --git a/fs/_url_tools.py b/fs/_url_tools.py index af55ff74..cfd76a7a 100644 --- a/fs/_url_tools.py +++ b/fs/_url_tools.py @@ -1,7 +1,8 @@ +import typing + +import platform import re import six -import platform -import typing if typing.TYPE_CHECKING: from typing import Text diff --git a/fs/appfs.py b/fs/appfs.py index 131ea8a8..2fd45687 100644 --- a/fs/appfs.py +++ b/fs/appfs.py @@ -9,14 +9,14 @@ # see http://technet.microsoft.com/en-us/library/cc766489(WS.10).aspx -import abc import typing +import abc import six +from appdirs import AppDirs -from .osfs import OSFS from ._repr import make_repr -from appdirs import AppDirs +from .osfs import OSFS if typing.TYPE_CHECKING: from typing import Optional, Text diff --git a/fs/base.py b/fs/base.py index 91152a60..07a16756 100644 --- a/fs/base.py +++ b/fs/base.py @@ -8,18 +8,18 @@ from __future__ import absolute_import, print_function, unicode_literals +import typing + import abc import hashlib import itertools import os +import six import threading import time -import typing +import warnings from contextlib import closing from functools import partial, wraps -import warnings - -import six from . import copy, errors, fsencode, iotools, tools, walk, wildcard from .copy import copy_modified_time @@ -30,15 +30,13 @@ from .walk import Walker if typing.TYPE_CHECKING: - from datetime import datetime - from threading import RLock from typing import ( + IO, Any, BinaryIO, Callable, Collection, Dict, - IO, Iterable, Iterator, List, @@ -49,11 +47,15 @@ Type, Union, ) + + from datetime import datetime + from threading import RLock from types import TracebackType + from .enums import ResourceType from .info import Info, RawInfo - from .subfs import SubFS from .permissions import Permissions + from .subfs import SubFS from .walk import BoundWalker _F = typing.TypeVar("_F", bound="FS") diff --git a/fs/compress.py b/fs/compress.py index a3d73033..a1b2e346 100644 --- a/fs/compress.py +++ b/fs/compress.py @@ -4,26 +4,25 @@ `tarfile` modules from the standard library. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals -import time -import tarfile import typing -import zipfile -from datetime import datetime import six +import tarfile +import time +import zipfile +from datetime import datetime from .enums import ResourceType +from .errors import MissingInfoNamespace, NoSysPath from .path import relpath from .time import datetime_to_epoch -from .errors import NoSysPath, MissingInfoNamespace from .walk import Walker if typing.TYPE_CHECKING: from typing import BinaryIO, Optional, Text, Tuple, Union + from .base import FS ZipTime = Tuple[int, int, int, int, int, int] diff --git a/fs/constants.py b/fs/constants.py index b9902a37..c12a9256 100644 --- a/fs/constants.py +++ b/fs/constants.py @@ -3,7 +3,6 @@ import io - DEFAULT_CHUNK_SIZE = io.DEFAULT_BUFFER_SIZE * 16 """`int`: the size of a single chunk read from or written to a file. """ diff --git a/fs/copy.py b/fs/copy.py index 6ffd83d7..90848c76 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -4,6 +4,7 @@ from __future__ import print_function, unicode_literals import typing + import warnings from .errors import ResourceNotFound @@ -14,6 +15,7 @@ if typing.TYPE_CHECKING: from typing import Callable, Optional, Text, Union + from .base import FS _OnCopy = Callable[[FS, Text, FS, Text], object] diff --git a/fs/enums.py b/fs/enums.py index 3c7d3ed0..adc288dd 100644 --- a/fs/enums.py +++ b/fs/enums.py @@ -1,8 +1,7 @@ """Enums used by PyFilesystem. """ -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals import os from enum import IntEnum, unique diff --git a/fs/error_tools.py b/fs/error_tools.py index 66d38696..bdb3818c 100644 --- a/fs/error_tools.py +++ b/fs/error_tools.py @@ -1,23 +1,23 @@ """Tools for managing OS errors. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import errno -import platform import sys import typing -from contextlib import contextmanager +import errno +import platform +from contextlib import contextmanager from six import reraise from . import errors if typing.TYPE_CHECKING: - from types import TracebackType from typing import Iterator, Optional, Text, Type, Union + from types import TracebackType + try: from collections.abc import Mapping except ImportError: diff --git a/fs/errors.py b/fs/errors.py index ba193bdb..400bac76 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -8,12 +8,11 @@ """ -from __future__ import unicode_literals -from __future__ import print_function +from __future__ import print_function, unicode_literals -import functools import typing +import functools import six from six import text_type diff --git a/fs/filesize.py b/fs/filesize.py index fafcc61d..ed113e88 100644 --- a/fs/filesize.py +++ b/fs/filesize.py @@ -11,8 +11,7 @@ """ -from __future__ import division -from __future__ import unicode_literals +from __future__ import division, unicode_literals import typing @@ -36,7 +35,7 @@ def _to_str(size, suffixes, base): # TODO (dargueta): Don't rely on unit or suffix being defined in the loop. for i, suffix in enumerate(suffixes, 2): # noqa: B007 - unit = base ** i + unit = base**i if size < unit: break return "{:,.1f} {}".format((base * size / unit), suffix) diff --git a/fs/ftpfs.py b/fs/ftpfs.py index b7a49988..50d8a0d5 100644 --- a/fs/ftpfs.py +++ b/fs/ftpfs.py @@ -1,8 +1,9 @@ """Manage filesystems on remote FTP servers. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals + +import typing import array import calendar @@ -11,7 +12,6 @@ import itertools import socket import threading -import typing from collections import OrderedDict from contextlib import contextmanager from ftplib import FTP @@ -20,41 +20,32 @@ from ftplib import FTP_TLS except ImportError as err: FTP_TLS = err # type: ignore -from ftplib import error_perm, error_temp from typing import cast -from six import PY2 -from six import text_type -from six import raise_from +from ftplib import error_perm, error_temp +from six import PY2, raise_from, text_type +from . import _ftp_parse as ftp_parse from . import errors from .base import FS from .constants import DEFAULT_CHUNK_SIZE -from .enums import ResourceType -from .enums import Seek +from .enums import ResourceType, Seek from .info import Info from .iotools import line_iterator from .mode import Mode -from .path import abspath -from .path import dirname -from .path import basename -from .path import normpath -from .path import split +from .path import abspath, basename, dirname, normpath, split from .time import epoch_to_datetime -from . import _ftp_parse as ftp_parse if typing.TYPE_CHECKING: - import mmap - import ftplib from typing import ( Any, BinaryIO, ByteString, + Container, ContextManager, + Dict, Iterable, Iterator, - Container, - Dict, List, Optional, SupportsInt, @@ -62,6 +53,10 @@ Tuple, Union, ) + + import ftplib + import mmap + from .base import _OpendirFactory from .info import RawInfo from .permissions import Permissions @@ -131,7 +126,6 @@ def _decode(st, encoding): # type: (Union[Text, bytes], Text) -> Text return st.decode(encoding, "replace") if isinstance(st, bytes) else st - else: def _encode(st, _): diff --git a/fs/glob.py b/fs/glob.py index 0dfa9f1c..cd6473d2 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -3,15 +3,15 @@ from __future__ import unicode_literals -from collections import namedtuple -import re import typing -from .lrucache import LRUCache +import re +from collections import namedtuple + +from . import wildcard from ._repr import make_repr +from .lrucache import LRUCache from .path import iteratepath -from . import wildcard - GlobMatch = namedtuple("GlobMatch", ["path", "info"]) Counts = namedtuple("Counts", ["files", "directories", "data"]) @@ -19,6 +19,7 @@ if typing.TYPE_CHECKING: from typing import Iterator, List, Optional, Pattern, Text, Tuple + from .base import FS diff --git a/fs/info.py b/fs/info.py index a2208cc5..21bb1498 100644 --- a/fs/info.py +++ b/fs/info.py @@ -1,27 +1,26 @@ """Container for filesystem resource informations. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing from typing import cast -from copy import deepcopy import six +from copy import deepcopy -from .path import join +from ._typing import Text, overload from .enums import ResourceType from .errors import MissingInfoNamespace +from .path import join from .permissions import Permissions from .time import epoch_to_datetime -from ._typing import overload, Text if typing.TYPE_CHECKING: - from datetime import datetime from typing import Any, Callable, List, Mapping, Optional, Union + from datetime import datetime + RawInfo = Mapping[Text, Mapping[Text, object]] ToDatetime = Callable[[int], datetime] T = typing.TypeVar("T") diff --git a/fs/iotools.py b/fs/iotools.py index bf7f37a5..fbef6fef 100644 --- a/fs/iotools.py +++ b/fs/iotools.py @@ -1,29 +1,21 @@ """Compatibility tools between Python 2 and Python 3 I/O interfaces. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals + +import typing import array import io -import typing -from io import SEEK_SET, SEEK_CUR +from io import SEEK_CUR, SEEK_SET from .mode import Mode if typing.TYPE_CHECKING: + from typing import IO, Any, Iterable, Iterator, List, Optional, Text, Union + import mmap from io import RawIOBase - from typing import ( - Any, - Iterable, - Iterator, - IO, - List, - Optional, - Text, - Union, - ) class RawWrapper(io.RawIOBase): diff --git a/fs/lrucache.py b/fs/lrucache.py index 9deb98bd..8ae26de5 100644 --- a/fs/lrucache.py +++ b/fs/lrucache.py @@ -1,12 +1,11 @@ """Least Recently Used cache mapping. """ -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals import typing -from collections import OrderedDict +from collections import OrderedDict _K = typing.TypeVar("_K") _V = typing.TypeVar("_V") diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 35ef0f51..8efdc139 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -1,32 +1,27 @@ """Manage a volatile in-memory filesystem. """ -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals + +import typing import contextlib import io import os +import six import time -import typing from collections import OrderedDict from threading import RLock -import six - from . import errors +from ._typing import overload from .base import FS from .copy import copy_modified_time from .enums import ResourceType, Seek from .info import Info from .mode import Mode -from .path import iteratepath -from .path import normpath -from .path import split -from ._typing import overload +from .path import iteratepath, normpath, split if typing.TYPE_CHECKING: - import array - import mmap from typing import ( Any, BinaryIO, @@ -37,10 +32,14 @@ List, Optional, SupportsInt, - Union, Text, Tuple, + Union, ) + + import array + import mmap + from .base import _OpendirFactory from .info import RawInfo from .permissions import Permissions diff --git a/fs/mirror.py b/fs/mirror.py index dd00ff7b..70b2dc5f 100644 --- a/fs/mirror.py +++ b/fs/mirror.py @@ -16,8 +16,7 @@ """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing @@ -30,6 +29,7 @@ if typing.TYPE_CHECKING: from typing import Callable, Optional, Text, Union + from .base import FS from .info import Info diff --git a/fs/mode.py b/fs/mode.py index 613e1ba9..c719340c 100644 --- a/fs/mode.py +++ b/fs/mode.py @@ -5,8 +5,7 @@ """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing @@ -14,7 +13,6 @@ from ._typing import Text - if typing.TYPE_CHECKING: from typing import FrozenSet, Set, Union diff --git a/fs/mountfs.py b/fs/mountfs.py index 5e590637..92fba6d7 100644 --- a/fs/mountfs.py +++ b/fs/mountfs.py @@ -1,9 +1,7 @@ """Manage other filesystems as a folder hierarchy. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing @@ -12,19 +10,16 @@ from . import errors from .base import FS from .memoryfs import MemoryFS -from .path import abspath -from .path import forcedir -from .path import normpath -from .mode import validate_open_mode -from .mode import validate_openbin_mode +from .mode import validate_open_mode, validate_openbin_mode +from .path import abspath, forcedir, normpath if typing.TYPE_CHECKING: from typing import ( + IO, Any, BinaryIO, Collection, Iterator, - IO, List, MutableSequence, Optional, @@ -32,6 +27,7 @@ Tuple, Union, ) + from .enums import ResourceType from .info import Info, RawInfo from .permissions import Permissions diff --git a/fs/move.py b/fs/move.py index 5b200963..fdbe96fe 100644 --- a/fs/move.py +++ b/fs/move.py @@ -1,23 +1,22 @@ """Functions for moving files between filesystems. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing -from .copy import copy_dir -from .copy import copy_file +from ._pathcompat import commonpath +from .copy import copy_dir, copy_file from .errors import FSError from .opener import manage_fs from .osfs import OSFS from .path import frombase -from ._pathcompat import commonpath if typing.TYPE_CHECKING: - from .base import FS from typing import Text, Union + from .base import FS + def move_fs( src_fs, # type: Union[Text, FS] diff --git a/fs/multifs.py b/fs/multifs.py index 6f0fff42..125d0de4 100644 --- a/fs/multifs.py +++ b/fs/multifs.py @@ -1,14 +1,12 @@ """Manage several filesystems through a single view. """ -from __future__ import absolute_import -from __future__ import unicode_literals -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals import typing -from collections import namedtuple, OrderedDict -from operator import itemgetter +from collections import OrderedDict, namedtuple +from operator import itemgetter from six import text_type from . import errors @@ -19,18 +17,19 @@ if typing.TYPE_CHECKING: from typing import ( + IO, Any, BinaryIO, Collection, Iterator, - IO, - MutableMapping, List, + MutableMapping, MutableSet, Optional, Text, Tuple, ) + from .enums import ResourceType from .info import Info, RawInfo from .permissions import Permissions diff --git a/fs/opener/__init__.py b/fs/opener/__init__.py index c20545eb..651a630b 100644 --- a/fs/opener/__init__.py +++ b/fs/opener/__init__.py @@ -5,14 +5,14 @@ # Declare fs.opener as a namespace package __import__("pkg_resources").declare_namespace(__name__) # type: ignore +# Import opener modules so that `registry.install` if called on each opener +from . import appfs, ftpfs, memoryfs, osfs, tarfs, tempfs, zipfs + # Import objects into fs.opener namespace from .base import Opener from .parse import parse_fs_url as parse from .registry import registry -# Import opener modules so that `registry.install` if called on each opener -from . import appfs, ftpfs, memoryfs, osfs, tarfs, tempfs, zipfs - # Alias functions defined as Registry methods open_fs = registry.open_fs open = registry.open diff --git a/fs/opener/appfs.py b/fs/opener/appfs.py index 0b1d78fa..db8bd34f 100644 --- a/fs/opener/appfs.py +++ b/fs/opener/appfs.py @@ -2,21 +2,20 @@ """``AppFS`` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing from .base import Opener -from .registry import registry from .errors import OpenerError +from .registry import registry if typing.TYPE_CHECKING: from typing import Text, Union - from .parse import ParseResult + from ..appfs import _AppFS from ..subfs import SubFS + from .parse import ParseResult @registry.install @@ -36,8 +35,8 @@ def open_fs( ): # type: (...) -> Union[_AppFS, SubFS[_AppFS]] - from ..subfs import ClosingSubFS from .. import appfs + from ..subfs import ClosingSubFS if self._protocol_mapping is None: self._protocol_mapping = { diff --git a/fs/opener/base.py b/fs/opener/base.py index cd970399..5facaaae 100644 --- a/fs/opener/base.py +++ b/fs/opener/base.py @@ -2,13 +2,14 @@ """`Opener` abstract base class. """ -import abc import typing +import abc import six if typing.TYPE_CHECKING: from typing import List, Text + from ..base import FS from .parse import ParseResult diff --git a/fs/opener/ftpfs.py b/fs/opener/ftpfs.py index cd313a34..6729ee1e 100644 --- a/fs/opener/ftpfs.py +++ b/fs/opener/ftpfs.py @@ -2,18 +2,17 @@ """`FTPFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing +from ..errors import CreateFailed from .base import Opener from .registry import registry -from ..errors import CreateFailed if typing.TYPE_CHECKING: from typing import Text, Union + from ..ftpfs import FTPFS # noqa: F401 from ..subfs import SubFS from .parse import ParseResult diff --git a/fs/opener/memoryfs.py b/fs/opener/memoryfs.py index 1ce8f105..400d2d9c 100644 --- a/fs/opener/memoryfs.py +++ b/fs/opener/memoryfs.py @@ -2,9 +2,7 @@ """`MemoryFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing @@ -13,8 +11,9 @@ if typing.TYPE_CHECKING: from typing import Text - from .parse import ParseResult + from ..memoryfs import MemoryFS # noqa: F401 + from .parse import ParseResult @registry.install diff --git a/fs/opener/osfs.py b/fs/opener/osfs.py index 7cb87b99..e9c3fc45 100644 --- a/fs/opener/osfs.py +++ b/fs/opener/osfs.py @@ -2,9 +2,7 @@ """`OSFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing @@ -13,8 +11,9 @@ if typing.TYPE_CHECKING: from typing import Text - from .parse import ParseResult + from ..osfs import OSFS # noqa: F401 + from .parse import ParseResult @registry.install @@ -32,8 +31,9 @@ def open_fs( cwd, # type: Text ): # type: (...) -> OSFS + from os.path import abspath, expanduser, join, normpath + from ..osfs import OSFS - from os.path import abspath, expanduser, normpath, join _path = abspath(join(cwd, expanduser(parse_result.resource))) path = normpath(_path) diff --git a/fs/opener/parse.py b/fs/opener/parse.py index e49a8009..f554bf38 100644 --- a/fs/opener/parse.py +++ b/fs/opener/parse.py @@ -1,14 +1,12 @@ """Function to parse FS URLs in to their constituent parts. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals -import collections -import re import typing +import collections +import re import six from six.moves.urllib.parse import parse_qs, unquote diff --git a/fs/opener/registry.py b/fs/opener/registry.py index 9b955549..19547234 100644 --- a/fs/opener/registry.py +++ b/fs/opener/registry.py @@ -2,32 +2,22 @@ """`Registry` class mapping protocols and FS URLs to their `Opener`. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals -import collections -import contextlib import typing +import collections +import contextlib import pkg_resources -from .base import Opener -from .errors import UnsupportedProtocol, EntryPointError from ..errors import ResourceReadOnly +from .base import Opener +from .errors import EntryPointError, UnsupportedProtocol from .parse import parse_fs_url if typing.TYPE_CHECKING: - from typing import ( - Callable, - Dict, - Iterator, - List, - Text, - Type, - Tuple, - Union, - ) + from typing import Callable, Dict, Iterator, List, Text, Tuple, Type, Union + from ..base import FS diff --git a/fs/opener/tarfs.py b/fs/opener/tarfs.py index bacb4e65..e53a51b9 100644 --- a/fs/opener/tarfs.py +++ b/fs/opener/tarfs.py @@ -2,20 +2,19 @@ """`TarFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing from .base import Opener -from .registry import registry from .errors import NotWriteable +from .registry import registry if typing.TYPE_CHECKING: from typing import Text - from .parse import ParseResult + from ..tarfs import TarFS # noqa: F401 + from .parse import ParseResult @registry.install diff --git a/fs/opener/tempfs.py b/fs/opener/tempfs.py index 22e26e0c..f48eb099 100644 --- a/fs/opener/tempfs.py +++ b/fs/opener/tempfs.py @@ -2,9 +2,7 @@ """`TempFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing @@ -13,8 +11,9 @@ if typing.TYPE_CHECKING: from typing import Text - from .parse import ParseResult + from ..tempfs import TempFS # noqa: F401 + from .parse import ParseResult @registry.install diff --git a/fs/opener/zipfs.py b/fs/opener/zipfs.py index dbc0fe7c..10c979cc 100644 --- a/fs/opener/zipfs.py +++ b/fs/opener/zipfs.py @@ -2,20 +2,19 @@ """`ZipFS` opener definition. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import typing from .base import Opener -from .registry import registry from .errors import NotWriteable +from .registry import registry if typing.TYPE_CHECKING: from typing import Text - from .parse import ParseResult + from ..zipfs import ZipFS # noqa: F401 + from .parse import ParseResult @registry.install diff --git a/fs/osfs.py b/fs/osfs.py index 3e6292ef..0add3d97 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -4,9 +4,10 @@ of the Python standard library. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals + +import sys +import typing import errno import io @@ -15,12 +16,9 @@ import os import platform import shutil +import six import stat -import sys import tempfile -import typing - -import six try: from os import scandir @@ -39,32 +37,33 @@ sendfile = None # type: ignore # pragma: no cover from . import errors +from ._fscompat import fsdecode, fsencode, fspath +from ._url_tools import url_quote from .base import FS +from .copy import copy_modified_time from .enums import ResourceType -from ._fscompat import fsencode, fsdecode, fspath +from .error_tools import convert_os_errors +from .errors import FileExpected, NoURL from .info import Info +from .mode import Mode, validate_open_mode from .path import basename, dirname from .permissions import Permissions -from .error_tools import convert_os_errors -from .mode import Mode, validate_open_mode -from .errors import FileExpected, NoURL -from ._url_tools import url_quote -from .copy import copy_modified_time if typing.TYPE_CHECKING: from typing import ( + IO, Any, BinaryIO, Collection, Dict, Iterator, - IO, List, Optional, SupportsInt, Text, Tuple, ) + from .base import _OpendirFactory from .info import RawInfo from .subfs import SubFS diff --git a/fs/path.py b/fs/path.py index 13641be1..0bfa5149 100644 --- a/fs/path.py +++ b/fs/path.py @@ -8,12 +8,12 @@ """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import re import typing +import re + from .errors import IllegalBackReference if typing.TYPE_CHECKING: diff --git a/fs/permissions.py b/fs/permissions.py index 3aaa6eff..3fee3352 100644 --- a/fs/permissions.py +++ b/fs/permissions.py @@ -1,8 +1,7 @@ """Abstract permissions container. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing from typing import Iterable @@ -11,7 +10,6 @@ from ._typing import Text - if typing.TYPE_CHECKING: from typing import Iterator, List, Optional, Tuple, Type, Union diff --git a/fs/subfs.py b/fs/subfs.py index 0582734a..9bc6167b 100644 --- a/fs/subfs.py +++ b/fs/subfs.py @@ -1,20 +1,20 @@ """Manage a directory in a *parent* filesystem. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing import six -from .wrapfs import WrapFS from .path import abspath, join, normpath, relpath +from .wrapfs import WrapFS if typing.TYPE_CHECKING: - from .base import FS # noqa: F401 from typing import Text, Tuple + from .base import FS # noqa: F401 + _F = typing.TypeVar("_F", bound="FS", covariant=True) diff --git a/fs/tarfs.py b/fs/tarfs.py index 0e808fe1..e699f86a 100644 --- a/fs/tarfs.py +++ b/fs/tarfs.py @@ -1,18 +1,18 @@ """Manage the filesystem in a Tar archive. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import os -import tarfile import typing -from collections import OrderedDict -from typing import cast, IO +from typing import IO, cast +import os import six +import tarfile +from collections import OrderedDict from . import errors +from ._url_tools import url_quote from .base import FS from .compress import write_tar from .enums import ResourceType @@ -20,13 +20,11 @@ from .info import Info from .iotools import RawWrapper from .opener import open_fs +from .path import basename, frombase, isbase, normpath, parts, relpath from .permissions import Permissions -from ._url_tools import url_quote -from .path import relpath, basename, isbase, normpath, parts, frombase from .wrapfs import WrapFS if typing.TYPE_CHECKING: - from tarfile import TarInfo from typing import ( Any, BinaryIO, @@ -38,6 +36,9 @@ Tuple, Union, ) + + from tarfile import TarInfo + from .info import RawInfo from .subfs import SubFS @@ -53,7 +54,6 @@ def _get_member_info(member, encoding): # type: (TarInfo, Text) -> Dict[Text, object] return member.get_info(encoding, None) - else: def _get_member_info(member, encoding): diff --git a/fs/tempfs.py b/fs/tempfs.py index 5fdc2f61..3f32c8c6 100644 --- a/fs/tempfs.py +++ b/fs/tempfs.py @@ -9,14 +9,13 @@ """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import shutil -import tempfile import typing +import shutil import six +import tempfile from . import errors from .osfs import OSFS diff --git a/fs/test.py b/fs/test.py index 0968ce75..232666da 100644 --- a/fs/test.py +++ b/fs/test.py @@ -5,30 +5,25 @@ """ -from __future__ import absolute_import -from __future__ import unicode_literals +from __future__ import absolute_import, unicode_literals -from datetime import datetime import io import itertools import json import os +import six import time import unittest import warnings +from datetime import datetime +from six import text_type import fs.copy import fs.move -from fs import ResourceType, Seek -from fs import errors -from fs import walk -from fs import glob +from fs import ResourceType, Seek, errors, glob, walk from fs.opener import open_fs from fs.subfs import ClosingSubFS, SubFS -import six -from six import text_type - if six.PY2: import collections as collections_abc else: diff --git a/fs/time.py b/fs/time.py index 52a59a77..b462c47f 100644 --- a/fs/time.py +++ b/fs/time.py @@ -1,10 +1,10 @@ """Time related tools. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing + from calendar import timegm from datetime import datetime diff --git a/fs/tools.py b/fs/tools.py index 7dac4d25..ca3058e4 100644 --- a/fs/tools.py +++ b/fs/tools.py @@ -1,21 +1,17 @@ """Miscellaneous tools for operating on filesystems. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing from . import errors -from .errors import DirectoryNotEmpty -from .errors import ResourceNotFound -from .path import abspath -from .path import dirname -from .path import normpath -from .path import recursepath +from .errors import DirectoryNotEmpty, ResourceNotFound +from .path import abspath, dirname, normpath, recursepath if typing.TYPE_CHECKING: from typing import IO, List, Optional, Text, Union + from .base import FS diff --git a/fs/tree.py b/fs/tree.py index 0f3142fe..598f2212 100644 --- a/fs/tree.py +++ b/fs/tree.py @@ -4,8 +4,7 @@ Color is supported on UNIX terminals. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import sys import typing @@ -14,6 +13,7 @@ if typing.TYPE_CHECKING: from typing import List, Optional, Text, TextIO, Tuple + from .base import FS from .info import Info diff --git a/fs/walk.py b/fs/walk.py index f539fa9d..31dc6b0e 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -8,15 +8,12 @@ from __future__ import unicode_literals import typing -from collections import defaultdict -from collections import deque -from collections import namedtuple + +from collections import defaultdict, deque, namedtuple from ._repr import make_repr from .errors import FSError -from .path import abspath -from .path import combine -from .path import normpath +from .path import abspath, combine, normpath if typing.TYPE_CHECKING: from typing import ( @@ -25,12 +22,13 @@ Collection, Iterator, List, - Optional, MutableMapping, + Optional, Text, Tuple, Type, ) + from .base import FS from .info import Info diff --git a/fs/wildcard.py b/fs/wildcard.py index a43a84b7..7a34d6b7 100644 --- a/fs/wildcard.py +++ b/fs/wildcard.py @@ -2,16 +2,17 @@ """ # Adapted from https://hg.python.org/cpython/file/2.7/Lib/fnmatch.py -from __future__ import unicode_literals, print_function +from __future__ import print_function, unicode_literals -import re import typing + +import re from functools import partial from .lrucache import LRUCache if typing.TYPE_CHECKING: - from typing import Callable, Iterable, Text, Tuple, Pattern + from typing import Callable, Iterable, Pattern, Text, Tuple _PATTERN_CACHE = LRUCache(1000) # type: LRUCache[Tuple[Text, bool], Pattern] diff --git a/fs/wrap.py b/fs/wrap.py index 113ca28f..de38d083 100644 --- a/fs/wrap.py +++ b/fs/wrap.py @@ -11,35 +11,36 @@ """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import typing -from .wrapfs import WrapFS -from .path import abspath, normpath, split -from .errors import ResourceReadOnly, ResourceNotFound +from .errors import ResourceNotFound, ResourceReadOnly from .info import Info from .mode import check_writable +from .path import abspath, normpath, split +from .wrapfs import WrapFS if typing.TYPE_CHECKING: - from datetime import datetime from typing import ( + IO, Any, BinaryIO, Collection, Dict, Iterator, - IO, Mapping, Optional, Text, Tuple, ) + + from datetime import datetime + from .base import FS # noqa: F401 from .info import RawInfo - from .subfs import SubFS from .permissions import Permissions + from .subfs import SubFS _W = typing.TypeVar("_W", bound="WrapFS") diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 00984e72..abbbe4e3 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -9,23 +9,21 @@ from . import errors from .base import FS -from .copy import copy_file, copy_dir +from .copy import copy_dir, copy_file +from .error_tools import unwrap_errors from .info import Info from .path import abspath, join, normpath -from .error_tools import unwrap_errors if typing.TYPE_CHECKING: - from datetime import datetime - from threading import RLock from typing import ( + IO, Any, AnyStr, BinaryIO, Callable, Collection, - Iterator, Iterable, - IO, + Iterator, List, Mapping, Optional, @@ -33,6 +31,10 @@ Tuple, Union, ) + + from datetime import datetime + from threading import RLock + from .enums import ResourceType from .info import RawInfo from .permissions import Permissions diff --git a/fs/zipfs.py b/fs/zipfs.py index 5c03754c..87e41f5e 100644 --- a/fs/zipfs.py +++ b/fs/zipfs.py @@ -1,29 +1,28 @@ """Manage the filesystem in a Zip archive. """ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import sys import typing -import zipfile -from datetime import datetime import six +import zipfile +from datetime import datetime from . import errors +from ._url_tools import url_quote from .base import FS from .compress import write_zip from .enums import ResourceType, Seek from .info import Info from .iotools import RawWrapper -from .permissions import Permissions from .memoryfs import MemoryFS from .opener import open_fs from .path import dirname, forcedir, normpath, relpath +from .permissions import Permissions from .time import datetime_to_epoch from .wrapfs import WrapFS -from ._url_tools import url_quote if typing.TYPE_CHECKING: from typing import ( @@ -38,6 +37,7 @@ Tuple, Union, ) + from .info import RawInfo from .subfs import SubFS diff --git a/setup.cfg b/setup.cfg index 154f7312..9245f143 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,7 +101,7 @@ per-file-ignores = [isort] default_section = THIRDPARTY known_first_party = fs -known_standard_library = typing +known_standard_library = sys, typing line_length = 88 profile = black skip_gitignore = true diff --git a/setup.py b/setup.py index 756c6bef..c4e2465a 100644 --- a/setup.py +++ b/setup.py @@ -6,4 +6,5 @@ exec(f.read()) from setuptools import setup + setup(version=__version__) diff --git a/tests/test_appfs.py b/tests/test_appfs.py index acc8a7f7..2d421482 100644 --- a/tests/test_appfs.py +++ b/tests/test_appfs.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals import shutil +import six import tempfile import unittest -import six - try: from unittest import mock except ImportError: diff --git a/tests/test_archives.py b/tests/test_archives.py index 0740e455..8b5397a2 100644 --- a/tests/test_archives.py +++ b/tests/test_archives.py @@ -3,13 +3,11 @@ import os import stat - from six import text_type -from fs.opener import open_fs +from fs import errors, walk from fs.enums import ResourceType -from fs import walk -from fs import errors +from fs.opener import open_fs from fs.test import UNICODE_TEXT diff --git a/tests/test_base.py b/tests/test_base.py index 937db27d..6bcb6639 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -4,8 +4,8 @@ import unittest -from fs.base import FS from fs import errors +from fs.base import FS class DummyFS(FS): diff --git a/tests/test_copy.py b/tests/test_copy.py index 6441e812..8e527648 100644 --- a/tests/test_copy.py +++ b/tests/test_copy.py @@ -1,13 +1,12 @@ from __future__ import unicode_literals -import errno +import calendar import datetime +import errno import os -import unittest -import tempfile import shutil -import calendar - +import tempfile +import unittest from parameterized import parameterized import fs.copy diff --git a/tests/test_doctest.py b/tests/test_doctest.py index 22c02357..ba27d82d 100644 --- a/tests/test_doctest.py +++ b/tests/test_doctest.py @@ -5,11 +5,11 @@ import importlib import os import pkgutil -import types -import warnings import tempfile import time +import types import unittest +import warnings from pprint import pprint try: diff --git a/tests/test_encoding.py b/tests/test_encoding.py index 0cd91d4c..6791e396 100644 --- a/tests/test_encoding.py +++ b/tests/test_encoding.py @@ -3,15 +3,13 @@ import os import platform import shutil +import six import tempfile import unittest -import six - import fs from fs.osfs import OSFS - if platform.system() != "Windows": @unittest.skipIf(platform.system() == "Darwin", "Bad unicode not possible on OSX") diff --git a/tests/test_enums.py b/tests/test_enums.py index fe496336..aa847c33 100644 --- a/tests/test_enums.py +++ b/tests/test_enums.py @@ -1,9 +1,8 @@ import os +import unittest from fs import enums -import unittest - class TestEnums(unittest.TestCase): def test_enums(self): diff --git a/tests/test_errors.py b/tests/test_errors.py index 5f4d8b8c..9688e345 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -2,7 +2,6 @@ import multiprocessing import unittest - from six import text_type from fs import errors diff --git a/tests/test_filesize.py b/tests/test_filesize.py index dc7b5af4..8900f671 100644 --- a/tests/test_filesize.py +++ b/tests/test_filesize.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals -from fs import filesize - import unittest +from fs import filesize + class TestFilesize(unittest.TestCase): def test_traditional(self): diff --git a/tests/test_fscompat.py b/tests/test_fscompat.py index 6418922b..d4544eab 100644 --- a/tests/test_fscompat.py +++ b/tests/test_fscompat.py @@ -1,10 +1,9 @@ from __future__ import unicode_literals -import unittest - import six +import unittest -from fs._fscompat import fsencode, fsdecode, fspath +from fs._fscompat import fsdecode, fsencode, fspath class PathMock(object): diff --git a/tests/test_ftpfs.py b/tests/test_ftpfs.py index d4143aa0..2bb2c73c 100644 --- a/tests/test_ftpfs.py +++ b/tests/test_ftpfs.py @@ -1,9 +1,8 @@ # coding: utf-8 -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import calendar +import datetime import os import platform import shutil @@ -12,23 +11,19 @@ import time import unittest import uuid -import datetime try: from unittest import mock except ImportError: import mock -from six import text_type, BytesIO - -from ftplib import error_perm -from ftplib import error_temp - +from ftplib import error_perm, error_temp from pyftpdlib.authorizers import DummyAuthorizer +from six import BytesIO, text_type from fs import errors -from fs.opener import open_fs from fs.ftpfs import FTPFS, ftp_errors +from fs.opener import open_fs from fs.path import join from fs.subfs import SubFS from fs.test import FSTestCases diff --git a/tests/test_glob.py b/tests/test_glob.py index c2a2d02f..51eb7779 100644 --- a/tests/test_glob.py +++ b/tests/test_glob.py @@ -2,8 +2,7 @@ import unittest -from fs import glob -from fs import open_fs +from fs import glob, open_fs class TestGlob(unittest.TestCase): diff --git a/tests/test_imports.py b/tests/test_imports.py index 8d8af34a..e18cffa7 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -1,4 +1,5 @@ import sys + import unittest diff --git a/tests/test_info.py b/tests/test_info.py index f6d6cfe9..7c50ec7b 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals -from datetime import datetime import unittest +from datetime import datetime from fs.enums import ResourceType from fs.info import Info diff --git a/tests/test_iotools.py b/tests/test_iotools.py index afb07d3c..56af6e73 100644 --- a/tests/test_iotools.py +++ b/tests/test_iotools.py @@ -1,13 +1,10 @@ from __future__ import unicode_literals import io -import unittest - import six +import unittest -from fs import iotools -from fs import tempfs - +from fs import iotools, tempfs from fs.test import UNICODE_TEXT diff --git a/tests/test_memoryfs.py b/tests/test_memoryfs.py index 67d92ac1..6537fac2 100644 --- a/tests/test_memoryfs.py +++ b/tests/test_memoryfs.py @@ -4,8 +4,7 @@ import unittest from fs import memoryfs -from fs.test import FSTestCases -from fs.test import UNICODE_TEXT +from fs.test import UNICODE_TEXT, FSTestCases try: # Only supported on Python 3.4+ diff --git a/tests/test_mirror.py b/tests/test_mirror.py index 1cce3d59..8aaa0953 100644 --- a/tests/test_mirror.py +++ b/tests/test_mirror.py @@ -1,11 +1,10 @@ from __future__ import unicode_literals import unittest - from parameterized import parameterized_class -from fs.mirror import mirror from fs import open_fs +from fs.mirror import mirror @parameterized_class(("WORKERS",), [(0,), (1,), (2,), (4,)]) diff --git a/tests/test_mode.py b/tests/test_mode.py index 86634f40..8fce62f2 100644 --- a/tests/test_mode.py +++ b/tests/test_mode.py @@ -1,10 +1,9 @@ from __future__ import unicode_literals import unittest - from six import text_type -from fs.mode import check_readable, check_writable, Mode +from fs.mode import Mode, check_readable, check_writable class TestMode(unittest.TestCase): diff --git a/tests/test_mountfs.py b/tests/test_mountfs.py index 1ffa82d2..f8403626 100644 --- a/tests/test_mountfs.py +++ b/tests/test_mountfs.py @@ -2,8 +2,8 @@ import unittest -from fs.mountfs import MountError, MountFS from fs.memoryfs import MemoryFS +from fs.mountfs import MountError, MountFS from fs.tempfs import TempFS from fs.test import FSTestCases diff --git a/tests/test_move.py b/tests/test_move.py index 2ec1b71b..5401082e 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -9,7 +9,6 @@ from parameterized import parameterized, parameterized_class - import fs.move from fs import open_fs from fs.errors import FSError, ResourceReadOnly diff --git a/tests/test_multifs.py b/tests/test_multifs.py index 7f0fe88b..623f5881 100644 --- a/tests/test_multifs.py +++ b/tests/test_multifs.py @@ -2,10 +2,9 @@ import unittest -from fs.multifs import MultiFS -from fs.memoryfs import MemoryFS from fs import errors - +from fs.memoryfs import MemoryFS +from fs.multifs import MultiFS from fs.test import FSTestCases diff --git a/tests/test_new_name.py b/tests/test_new_name.py index 647a96e8..5f571df9 100644 --- a/tests/test_new_name.py +++ b/tests/test_new_name.py @@ -3,7 +3,6 @@ import unittest import warnings - from fs.base import _new_name diff --git a/tests/test_opener.py b/tests/test_opener.py index bc2f5cd7..43d56903 100644 --- a/tests/test_opener.py +++ b/tests/test_opener.py @@ -1,19 +1,20 @@ from __future__ import unicode_literals +import sys + import os +import pkg_resources import shutil -import sys import tempfile import unittest -import pkg_resources from fs import open_fs, opener -from fs.osfs import OSFS -from fs.opener import registry, errors -from fs.memoryfs import MemoryFS from fs.appfs import UserDataFS +from fs.memoryfs import MemoryFS +from fs.opener import errors, registry from fs.opener.parse import ParseResult from fs.opener.registry import Registry +from fs.osfs import OSFS try: from unittest import mock diff --git a/tests/test_osfs.py b/tests/test_osfs.py index 7fdc5590..c77016a9 100644 --- a/tests/test_osfs.py +++ b/tests/test_osfs.py @@ -1,23 +1,22 @@ # coding: utf-8 from __future__ import unicode_literals +import sys + import errno import io import os import shutil import tempfile -import sys import time import unittest import warnings +from six import text_type -from fs import osfs, open_fs -from fs.path import relpath, dirname -from fs import errors +from fs import errors, open_fs, osfs +from fs.path import dirname, relpath from fs.test import FSTestCases -from six import text_type - try: from unittest import mock except ImportError: diff --git a/tests/test_path.py b/tests/test_path.py index 52673f03..3ab778ed 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals """ fstests.test_path: testcases for the fs path functions diff --git a/tests/test_permissions.py b/tests/test_permissions.py index 572ef8f3..72e5f197 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,11 +1,9 @@ -from __future__ import unicode_literals -from __future__ import print_function +from __future__ import print_function, unicode_literals import unittest - from six import text_type -from fs.permissions import make_mode, Permissions +from fs.permissions import Permissions, make_mode class TestPermissions(unittest.TestCase): diff --git a/tests/test_subfs.py b/tests/test_subfs.py index c3604312..494108cf 100644 --- a/tests/test_subfs.py +++ b/tests/test_subfs.py @@ -6,9 +6,10 @@ import unittest from fs import osfs -from fs.subfs import SubFS from fs.memoryfs import MemoryFS from fs.path import relpath +from fs.subfs import SubFS + from .test_osfs import TestOSFS diff --git a/tests/test_tarfs.py b/tests/test_tarfs.py index fc3f0779..29d23877 100644 --- a/tests/test_tarfs.py +++ b/tests/test_tarfs.py @@ -9,11 +9,11 @@ import unittest from fs import tarfs -from fs.enums import ResourceType from fs.compress import write_tar +from fs.enums import ResourceType +from fs.errors import NoURL from fs.opener import open_fs from fs.opener.errors import NotWriteable -from fs.errors import NoURL from fs.test import FSTestCases from .test_archives import ArchiveTestCases diff --git a/tests/test_tempfs.py b/tests/test_tempfs.py index f47c6129..eef46f76 100644 --- a/tests/test_tempfs.py +++ b/tests/test_tempfs.py @@ -2,8 +2,8 @@ import os -from fs.tempfs import TempFS from fs import errors +from fs.tempfs import TempFS from .test_osfs import TestOSFS diff --git a/tests/test_time.py b/tests/test_time.py index 5933e888..86a5972c 100644 --- a/tests/test_time.py +++ b/tests/test_time.py @@ -1,7 +1,7 @@ -from __future__ import unicode_literals, print_function +from __future__ import print_function, unicode_literals -from datetime import datetime import unittest +from datetime import datetime from fs.time import datetime_to_epoch, epoch_to_datetime diff --git a/tests/test_tools.py b/tests/test_tools.py index a151aac1..51d32963 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -2,9 +2,8 @@ import unittest -from fs.mode import validate_open_mode -from fs.mode import validate_openbin_mode from fs import tools +from fs.mode import validate_open_mode, validate_openbin_mode from fs.opener import open_fs diff --git a/tests/test_tree.py b/tests/test_tree.py index 2a4f942c..28f20577 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,5 +1,4 @@ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals import io import unittest diff --git a/tests/test_walk.py b/tests/test_walk.py index 8bf57dcf..5e05d054 100644 --- a/tests/test_walk.py +++ b/tests/test_walk.py @@ -1,12 +1,12 @@ from __future__ import unicode_literals +import six import unittest +from fs import walk from fs.errors import FSError from fs.memoryfs import MemoryFS -from fs import walk from fs.wrap import read_only -import six class TestWalker(unittest.TestCase): diff --git a/tests/test_wrap.py b/tests/test_wrap.py index e11ded4a..2438f2e0 100644 --- a/tests/test_wrap.py +++ b/tests/test_wrap.py @@ -11,10 +11,10 @@ import six import fs.copy +import fs.errors import fs.mirror import fs.move import fs.wrap -import fs.errors from fs import open_fs from fs.info import Info diff --git a/tests/test_wrapfs.py b/tests/test_wrapfs.py index d8d6a6b4..b4842de7 100644 --- a/tests/test_wrapfs.py +++ b/tests/test_wrapfs.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import unittest - from six import text_type from fs import wrapfs diff --git a/tests/test_zipfs.py b/tests/test_zipfs.py index 9b2e82ea..7390649c 100644 --- a/tests/test_zipfs.py +++ b/tests/test_zipfs.py @@ -1,21 +1,21 @@ # -*- encoding: UTF-8 from __future__ import unicode_literals -import os import sys + +import os +import six import tempfile import unittest import zipfile -import six - from fs import zipfs from fs.compress import write_zip +from fs.enums import Seek +from fs.errors import NoURL from fs.opener import open_fs from fs.opener.errors import NotWriteable -from fs.errors import NoURL from fs.test import FSTestCases -from fs.enums import Seek from .test_archives import ArchiveTestCases From 889765bd813f744dd8c2c8023d82be43a178a8cc Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 2 May 2022 10:32:45 +0200 Subject: [PATCH 279/309] Bump `black` version used to lint code in `setup.cfg` --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9245f143..57c6f40b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -169,7 +169,7 @@ deps = [testenv:codeformat] commands = black --check {toxinidir}/fs deps = - black==20.8b1 + black==22.3.0 [testenv:docstyle] commands = pydocstyle --config={toxinidir}/setup.cfg {toxinidir}/fs From 50b1c9956ee0c674c93ffbf675a7b0f58db2564d Mon Sep 17 00:00:00 2001 From: Martin Larralde Date: Mon, 2 May 2022 11:21:45 +0200 Subject: [PATCH 280/309] Release v2.4.16 --- CHANGELOG.md | 3 +++ fs/_version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9325628..267c942b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + +## [2.4.16] - 2022-05-02 + ### Changed - Make `fs.zipfs._ZipExtFile` use the seeking mechanism implemented diff --git a/fs/_version.py b/fs/_version.py index 188c0e14..c1ab7f71 100644 --- a/fs/_version.py +++ b/fs/_version.py @@ -1,3 +1,3 @@ """Version, used in module and setup.py. """ -__version__ = "2.4.15" +__version__ = "2.4.16" From a844a9bac32583911ed842229b3e6a8070c05fb6 Mon Sep 17 00:00:00 2001 From: atollk Date: Sat, 24 Jul 2021 11:22:13 +0200 Subject: [PATCH 281/309] Added filter_glob and exclude_glob to fs.walk.Walker. These extend the class by an option to include/exclude resources by their entire path, not just its last component. To do so, fs.wildcard had to undergo a rework to remove the dependency on the `re` module. Unit tests were added for all new/changed code. --- CHANGELOG.md | 9 ++ fs/base.py | 61 ++++++++- fs/errors.py | 17 +++ fs/glob.py | 179 ++++++++++++++++++++++++- fs/walk.py | 62 +++++++-- fs/wildcard.py | 14 +- tests/test_glob.py | 7 +- tests/test_walk.py | 319 ++++++++++++++++++++++++++++++--------------- 8 files changed, 536 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 267c942b..2c370c9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased +### Added + +- To `fs.walk.Walker`, added parameters `filter_glob` and `exclude_glob`. + Closes [#459](https://github.com/PyFilesystem/pyfilesystem2/issues/459). + +### Fixed +- Elaborated documentation of `filter_dirs` and `exclude_dirs` in `fs.walk.Walker`. + Closes [#371](https://github.com/PyFilesystem/pyfilesystem2/issues/371). + ## [2.4.16] - 2022-05-02 diff --git a/fs/base.py b/fs/base.py index 07a16756..dfbc6106 100644 --- a/fs/base.py +++ b/fs/base.py @@ -21,7 +21,7 @@ from contextlib import closing from functools import partial, wraps -from . import copy, errors, fsencode, iotools, tools, walk, wildcard +from . import copy, errors, fsencode, iotools, tools, walk, wildcard, glob from .copy import copy_modified_time from .glob import BoundGlobber from .mode import validate_open_mode @@ -1653,8 +1653,8 @@ def check(self): if self.isclosed(): raise errors.FilesystemClosed() - def match(self, patterns, name): - # type: (Optional[Iterable[Text]], Text) -> bool + def match(self, patterns, name, accept_prefix=False): + # type: (Optional[Iterable[Text]], Text, bool) -> bool """Check if a name matches any of a list of wildcards. If a filesystem is case *insensitive* (such as Windows) then @@ -1696,6 +1696,61 @@ def match(self, patterns, name): matcher = wildcard.get_matcher(patterns, case_sensitive) return matcher(name) + def match_glob(self, patterns, path, accept_prefix=False): + # type: (Optional[Iterable[Text]], Text, bool) -> bool + """Check if a path matches any of a list of glob patterns. + + If a filesystem is case *insensitive* (such as Windows) then + this method will perform a case insensitive match (i.e. ``*.py`` + will match the same names as ``*.PY``). Otherwise the match will + be case sensitive (``*.py`` and ``*.PY`` will match different + names). + + Arguments: + patterns (list, optional): A list of patterns, e.g. + ``['*.py']``, or `None` to match everything. + path (str): A resource path, starting with "/". + accept_prefix (bool): If ``True``, the path is + not required to match the wildcards themselves + but only need to be a prefix of a string that does. + + Returns: + bool: `True` if ``path`` matches any of the patterns. + + Raises: + TypeError: If ``patterns`` is a single string instead of + a list (or `None`). + ValueError: If ``path`` is not a string starting with "/". + + Example: + >>> my_fs.match_glob(['*.py'], '/__init__.py') + True + >>> my_fs.match_glob(['*.jpg', '*.png'], '/foo.gif') + False + >>> my_fs.match_glob(['dir/file.txt'], '/dir/', accept_prefix=True) + True + >>> my_fs.match_glob(['dir/file.txt'], '/dir/gile.txt', accept_prefix=True) + False + + Note: + If ``patterns`` is `None` (or ``['*']``), then this + method will always return `True`. + + """ + if patterns is None: + return True + if not path or path[0] != "/": + raise ValueError("%s needs to be a string starting with /" % path) + if isinstance(patterns, six.text_type): + raise TypeError("patterns must be a list or sequence") + case_sensitive = not typing.cast( + bool, self.getmeta().get("case_insensitive", False) + ) + matcher = glob.get_matcher( + patterns, case_sensitive, accept_prefix=accept_prefix + ) + return matcher(path) + def tree(self, **kwargs): # type: (**Any) -> None """Render a tree view of the filesystem to stdout or a file. diff --git a/fs/errors.py b/fs/errors.py index 400bac76..32c795d9 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -41,6 +41,7 @@ "OperationFailed", "OperationTimeout", "PathError", + "PatternError", "PermissionDenied", "RemoteConnectionError", "RemoveRootError", @@ -346,3 +347,19 @@ class UnsupportedHash(ValueError): not supported by hashlib. """ + + +class PatternError(ValueError): + """A string pattern with invalid syntax was given.""" + + default_message = "pattern '{pattern}' is invalid at position {position}" + + def __init__(self, pattern, position, exc=None, msg=None): # noqa: D107 + # type: (Text, int, Optional[Exception], Optional[Text]) -> None + self.pattern = pattern + self.position = position + self.exc = exc + super(ValueError, self).__init__() + + def __reduce__(self): + return type(self), (self.path, self.position, self.exc, self._msg) diff --git a/fs/glob.py b/fs/glob.py index cd6473d2..82eb0228 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -4,22 +4,31 @@ from __future__ import unicode_literals import typing +from functools import partial import re from collections import namedtuple -from . import wildcard from ._repr import make_repr from .lrucache import LRUCache from .path import iteratepath + GlobMatch = namedtuple("GlobMatch", ["path", "info"]) Counts = namedtuple("Counts", ["files", "directories", "data"]) LineCounts = namedtuple("LineCounts", ["lines", "non_blank"]) if typing.TYPE_CHECKING: - from typing import Iterator, List, Optional, Pattern, Text, Tuple - + from typing import ( + Iterator, + List, + Optional, + Pattern, + Text, + Tuple, + Iterable, + Callable, + ) from .base import FS @@ -28,17 +37,87 @@ ) # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]] +def _split_pattern_by_rec(pattern): + # type: (Text) -> List[Text] + """Split a glob pattern at its directory seperators (/). + + Takes into account escaped cases like [/]. + """ + indices = [-1] + bracket_open = False + for i, c in enumerate(pattern): + if c == "/" and not bracket_open: + indices.append(i) + elif c == "[": + bracket_open = True + elif c == "]": + bracket_open = False + + indices.append(len(pattern)) + return [pattern[i + 1 : j] for i, j in zip(indices[:-1], indices[1:])] + + +def _translate(pattern, case_sensitive=True): + # type: (Text, bool) -> Text + """Translate a wildcard pattern to a regular expression. + + There is no way to quote meta-characters. + Arguments: + pattern (str): A wildcard pattern. + case_sensitive (bool): Set to `False` to use a case + insensitive regex (default `True`). + + Returns: + str: A regex equivalent to the given pattern. + + """ + if not case_sensitive: + pattern = pattern.lower() + i, n = 0, len(pattern) + res = [] + while i < n: + c = pattern[i] + i = i + 1 + if c == "*": + res.append("[^/]*") + elif c == "?": + res.append("[^/]") + elif c == "[": + j = i + if j < n and pattern[j] == "!": + j = j + 1 + if j < n and pattern[j] == "]": + j = j + 1 + while j < n and pattern[j] != "]": + j = j + 1 + if j >= n: + res.append("\\[") + else: + stuff = pattern[i:j].replace("\\", "\\\\") + i = j + 1 + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] == "^": + stuff = "\\" + stuff + res.append("[%s]" % stuff) + else: + res.append(re.escape(c)) + return "".join(res) + + def _translate_glob(pattern, case_sensitive=True): levels = 0 recursive = False re_patterns = [""] for component in iteratepath(pattern): - if component == "**": - re_patterns.append(".*/?") + if "**" in component: recursive = True + split = component.split("**") + split_re = [_translate(s, case_sensitive=case_sensitive) for s in split] + re_patterns.append("/?" + ".*/?".join(split_re)) else: re_patterns.append( - "/" + wildcard._translate(component, case_sensitive=case_sensitive) + "/" + _translate(component, case_sensitive=case_sensitive) ) levels += 1 re_glob = "(?ms)^" + "".join(re_patterns) + ("/$" if pattern.endswith("/") else "$") @@ -72,6 +151,8 @@ def match(pattern, path): except KeyError: levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True) _PATTERN_CACHE[(pattern, True)] = (levels, recursive, re_pattern) + if path and path[0] != "/": + path = "/" + path return bool(re_pattern.match(path)) @@ -92,9 +173,95 @@ def imatch(pattern, path): except KeyError: levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True) _PATTERN_CACHE[(pattern, False)] = (levels, recursive, re_pattern) + if path and path[0] != "/": + path = "/" + path return bool(re_pattern.match(path)) +def match_any(patterns, path): + # type: (Iterable[Text], Text) -> bool + """Test if a path matches any of a list of patterns. + + Will return `True` if ``patterns`` is an empty list. + + Arguments: + patterns (list): A list of wildcard pattern, e.g ``["*.py", + "*.pyc"]`` + name (str): A filename. + + Returns: + bool: `True` if the path matches at least one of the patterns. + + """ + if not patterns: + return True + return any(match(pattern, path) for pattern in patterns) + + +def imatch_any(patterns, path): + # type: (Iterable[Text], Text) -> bool + """Test if a path matches any of a list of patterns (case insensitive). + + Will return `True` if ``patterns`` is an empty list. + + Arguments: + patterns (list): A list of wildcard pattern, e.g ``["*.py", + "*.pyc"]`` + name (str): A filename. + + Returns: + bool: `True` if the path matches at least one of the patterns. + + """ + if not patterns: + return True + return any(imatch(pattern, path) for pattern in patterns) + + +def get_matcher(patterns, case_sensitive, accept_prefix=False): + # type: (Iterable[Text], bool, bool) -> Callable[[Text], bool] + """Get a callable that matches paths against the given patterns. + + Arguments: + patterns (list): A list of wildcard pattern. e.g. ``["*.py", + "*.pyc"]`` + case_sensitive (bool): If ``True``, then the callable will be case + sensitive, otherwise it will be case insensitive. + accept_prefix (bool): If ``True``, the name is + not required to match the wildcards themselves + but only need to be a prefix of a string that does. + + Returns: + callable: a matcher that will return `True` if the paths given as + an argument matches any of the given patterns. + + Example: + >>> from fs import wildcard + >>> is_python = wildcard.get_matcher(['*.py'], True) + >>> is_python('__init__.py') + True + >>> is_python('foo.txt') + False + + """ + if not patterns: + return lambda name: True + + if accept_prefix: + new_patterns = [] + for pattern in patterns: + split = _split_pattern_by_rec(pattern) + for i in range(1, len(split)): + new_pattern = "/".join(split[:i]) + new_patterns.append(new_pattern) + new_patterns.append(new_pattern + "/") + new_patterns.append(pattern) + patterns = new_patterns + + matcher = match_any if case_sensitive else imatch_any + return partial(matcher, patterns) + + class Globber(object): """A generator of glob results.""" diff --git a/fs/walk.py b/fs/walk.py index 31dc6b0e..1724dfcc 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -60,6 +60,8 @@ def __init__( filter_dirs=None, # type: Optional[List[Text]] exclude_dirs=None, # type: Optional[List[Text]] max_depth=None, # type: Optional[int] + filter_glob=None, # type: Optional[List[Text]] + exclude_glob=None, # type: Optional[List[Text]] ): # type: (...) -> None """Create a new `Walker` instance. @@ -83,11 +85,22 @@ def __init__( any of these patterns will be removed from the walk. filter_dirs (list, optional): A list of patterns that will be used to match directories paths. The walk will only open directories - that match at least one of these patterns. + that match at least one of these patterns. Directories will + only be returned if the final component matches one of the + patterns. exclude_dirs (list, optional): A list of patterns that will be used to filter out directories from the walk. e.g. - ``['*.svn', '*.git']``. + ``['*.svn', '*.git']``. Directories matching any of these + patterns will be removed from the walk. max_depth (int, optional): Maximum directory depth to walk. + filter_glob (list, optional): If supplied, this parameter + should be a list of path patterns e.g. ``["foo/**/*.py"]``. + Resources will only be returned if their global path or + an extension of it matches one of the patterns. + exclude_glob (list, optional): If supplied, this parameter + should be a list of path patterns e.g. ``["foo/**/*.py"]``. + Resources will not be returned if their global path or + an extension of it matches one of the patterns. """ if search not in ("breadth", "depth"): @@ -107,6 +120,8 @@ def __init__( self.exclude = exclude self.filter_dirs = filter_dirs self.exclude_dirs = exclude_dirs + self.filter_glob = filter_glob + self.exclude_glob = exclude_glob self.max_depth = max_depth super(Walker, self).__init__() @@ -178,6 +193,8 @@ def __repr__(self): filter_dirs=(self.filter_dirs, None), exclude_dirs=(self.exclude_dirs, None), max_depth=(self.max_depth, None), + filter_glob=(self.filter_glob, None), + exclude_glob=(self.exclude_glob, None), ) def _iter_walk( @@ -196,10 +213,19 @@ def _iter_walk( def _check_open_dir(self, fs, path, info): # type: (FS, Text, Info) -> bool """Check if a directory should be considered in the walk.""" + full_path = ("" if path == "/" else path) + "/" + info.name if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name): return False + if self.exclude_glob is not None and fs.match_glob( + self.exclude_glob, full_path + ): + return False if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name): return False + if self.filter_glob is not None and not fs.match_glob( + self.filter_glob, full_path, accept_prefix=True + ): + return False return self.check_open_dir(fs, path, info) def check_open_dir(self, fs, path, info): @@ -245,6 +271,26 @@ def check_scan_dir(self, fs, path, info): """ return True + def _check_file(self, fs, dir_path, info): + # type: (FS, Text, Info) -> bool + """Check if a filename should be included.""" + # Weird check required for backwards compatibility, + # when _check_file did not exist. + if Walker._check_file == type(self)._check_file: + if self.exclude is not None and fs.match(self.exclude, info.name): + return False + if self.exclude_glob is not None and fs.match_glob( + self.exclude_glob, dir_path + "/" + info.name + ): + return False + if self.filter is not None and not fs.match(self.filter, info.name): + return False + if self.filter_glob is not None and not fs.match_glob( + self.filter_glob, dir_path + "/" + info.name, accept_prefix=True + ): + return False + return self.check_file(fs, info) + def check_file(self, fs, info): # type: (FS, Info) -> bool """Check if a filename should be included. @@ -259,9 +305,7 @@ def check_file(self, fs, info): bool: `True` if the file should be included. """ - if self.exclude is not None and fs.match(self.exclude, info.name): - return False - return fs.match(self.filter, info.name) + return True def _scan( self, @@ -418,7 +462,7 @@ def _walk_breadth( _calculate_depth = self._calculate_depth _check_open_dir = self._check_open_dir _check_scan_dir = self._check_scan_dir - _check_file = self.check_file + _check_file = self._check_file depth = _calculate_depth(path) @@ -432,7 +476,7 @@ def _walk_breadth( if _check_scan_dir(fs, dir_path, info, _depth): push(_combine(dir_path, info.name)) else: - if _check_file(fs, info): + if _check_file(fs, dir_path, info): yield dir_path, info # Found a file yield dir_path, None # End of directory @@ -451,7 +495,7 @@ def _walk_depth( _calculate_depth = self._calculate_depth _check_open_dir = self._check_open_dir _check_scan_dir = self._check_scan_dir - _check_file = self.check_file + _check_file = self._check_file depth = _calculate_depth(path) stack = [ @@ -483,7 +527,7 @@ def _walk_depth( else: yield dir_path, info else: - if _check_file(fs, info): + if _check_file(fs, dir_path, info): yield dir_path, info diff --git a/fs/wildcard.py b/fs/wildcard.py index 7a34d6b7..cc6c0530 100644 --- a/fs/wildcard.py +++ b/fs/wildcard.py @@ -147,14 +147,14 @@ def _translate(pattern, case_sensitive=True): if not case_sensitive: pattern = pattern.lower() i, n = 0, len(pattern) - res = "" + res = [] while i < n: c = pattern[i] i = i + 1 if c == "*": - res = res + "[^/]*" + res.append("[^/]*") elif c == "?": - res = res + "." + res.append(".") elif c == "[": j = i if j < n and pattern[j] == "!": @@ -164,7 +164,7 @@ def _translate(pattern, case_sensitive=True): while j < n and pattern[j] != "]": j = j + 1 if j >= n: - res = res + "\\[" + res.append("\\[") else: stuff = pattern[i:j].replace("\\", "\\\\") i = j + 1 @@ -172,7 +172,7 @@ def _translate(pattern, case_sensitive=True): stuff = "^" + stuff[1:] elif stuff[0] == "^": stuff = "\\" + stuff - res = "%s[%s]" % (res, stuff) + res.append("[%s]" % stuff) else: - res = res + re.escape(c) - return res + res.append(re.escape(c)) + return "".join(res) diff --git a/tests/test_glob.py b/tests/test_glob.py index 51eb7779..6147bf49 100644 --- a/tests/test_glob.py +++ b/tests/test_glob.py @@ -21,6 +21,7 @@ def test_match(self): tests = [ ("*.?y", "/test.py", True), ("*.py", "/test.py", True), + ("*.py", "__init__.py", True), ("*.py", "/test.pc", False), ("*.py", "/foo/test.py", False), ("foo/*.py", "/foo/test.py", True), @@ -28,21 +29,23 @@ def test_match(self): ("?oo/*.py", "/foo/test.py", True), ("*/*.py", "/foo/test.py", True), ("foo/*.py", "/bar/foo/test.py", False), + ("/foo/**", "/foo/test.py", True), ("**/foo/*.py", "/bar/foo/test.py", True), ("foo/**/bar/*.py", "/foo/bar/test.py", True), ("foo/**/bar/*.py", "/foo/baz/egg/bar/test.py", True), ("foo/**/bar/*.py", "/foo/baz/egg/bar/egg/test.py", False), ("**", "/test.py", True), + ("/**", "/test.py", True), ("**", "/test", True), ("**", "/test/", True), ("**/", "/test/", True), ("**/", "/test.py", False), ] for pattern, path, expected in tests: - self.assertEqual(glob.match(pattern, path), expected) + self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) # Run a second time to test cache for pattern, path, expected in tests: - self.assertEqual(glob.match(pattern, path), expected) + self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) def test_count_1dir(self): globber = glob.BoundGlobber(self.fs) diff --git a/tests/test_walk.py b/tests/test_walk.py index 5e05d054..e0146f2d 100644 --- a/tests/test_walk.py +++ b/tests/test_walk.py @@ -21,9 +21,28 @@ def test_create(self): walk.Walker(ignore_errors=True, on_error=lambda path, error: True) walk.Walker(ignore_errors=True) + def test_on_error_invalid(self): + with self.assertRaises(TypeError): + walk.Walker(on_error="nope") + -class TestWalk(unittest.TestCase): +class TestBoundWalkerBase(unittest.TestCase): def setUp(self): + """ + Sets up the following file system with empty files: + + / + -foo1/ + - -top1.txt + - -top2.txt + -foo2/ + - -bar1/ + - -bar2/ + - - -bar3/ + - - - -test.txt + - -top3.bin + -foo3/ + """ self.fs = MemoryFS() self.fs.makedir("foo1") @@ -37,21 +56,50 @@ def setUp(self): self.fs.create("foo2/bar2/bar3/test.txt") self.fs.create("foo2/top3.bin") - def test_invalid(self): - with self.assertRaises(ValueError): - self.fs.walk(search="random") +class TestBoundWalker(TestBoundWalkerBase): def test_repr(self): repr(self.fs.walk) - def test_walk(self): + def test_readonly_wrapper_uses_same_walker(self): + class CustomWalker(walk.Walker): + @classmethod + def bind(cls, fs): + return walk.BoundWalker(fs, walker_class=CustomWalker) + + class CustomizedMemoryFS(MemoryFS): + walker_class = CustomWalker + + base_fs = CustomizedMemoryFS() + base_walker = base_fs.walk + self.assertEqual(base_walker.walker_class, CustomWalker) + + readonly_fs = read_only(CustomizedMemoryFS()) + readonly_walker = readonly_fs.walk + self.assertEqual(readonly_walker.walker_class, CustomWalker) + + +class TestWalk(TestBoundWalkerBase): + def _walk_step_names(self, *args, **kwargs): + """Performs a walk with the given arguments and returns a list of steps. + + Each step is a triple of the path, list of directory names, and list of file names. + """ _walk = [] - for step in self.fs.walk(): + for step in self.fs.walk(*args, **kwargs): self.assertIsInstance(step, walk.Step) path, dirs, files = step _walk.append( (path, [info.name for info in dirs], [info.name for info in files]) ) + return _walk + + def test_invalid_search(self): + with self.assertRaises(ValueError): + self.fs.walk(search="random") + + def test_walk_simple(self): + _walk = self._walk_step_names() expected = [ ("/", ["foo1", "foo2", "foo3"], []), ("/foo1", ["bar1"], ["top1.txt", "top2.txt"]), @@ -63,14 +111,34 @@ def test_walk(self): ] self.assertEqual(_walk, expected) + def test_walk_filter(self): + _walk = self._walk_step_names(filter=["top*.txt"]) + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ("/foo1", ["bar1"], ["top1.txt", "top2.txt"]), + ("/foo2", ["bar2"], []), + ("/foo3", [], []), + ("/foo1/bar1", [], []), + ("/foo2/bar2", ["bar3"], []), + ("/foo2/bar2/bar3", [], []), + ] + self.assertEqual(_walk, expected) + + def test_walk_exclude(self): + _walk = self._walk_step_names(exclude=["top*"]) + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ("/foo1", ["bar1"], []), + ("/foo2", ["bar2"], []), + ("/foo3", [], []), + ("/foo1/bar1", [], []), + ("/foo2/bar2", ["bar3"], []), + ("/foo2/bar2/bar3", [], ["test.txt"]), + ] + self.assertEqual(_walk, expected) + def test_walk_filter_dirs(self): - _walk = [] - for step in self.fs.walk(filter_dirs=["foo*"]): - self.assertIsInstance(step, walk.Step) - path, dirs, files = step - _walk.append( - (path, [info.name for info in dirs], [info.name for info in files]) - ) + _walk = self._walk_step_names(filter_dirs=["foo*"]) expected = [ ("/", ["foo1", "foo2", "foo3"], []), ("/foo1", [], ["top1.txt", "top2.txt"]), @@ -79,14 +147,65 @@ def test_walk_filter_dirs(self): ] self.assertEqual(_walk, expected) + def test_walk_filter_glob_1(self): + _walk = self._walk_step_names(filter_glob=["/foo*/bar*/"]) + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ("/foo1", ["bar1"], []), + ("/foo2", ["bar2"], []), + ("/foo3", [], []), + ("/foo1/bar1", [], []), + ("/foo2/bar2", [], []), + ] + self.assertEqual(_walk, expected) + + def test_walk_filter_glob_2(self): + _walk = self._walk_step_names(filter_glob=["/foo*/bar**"]) + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ("/foo1", ["bar1"], []), + ("/foo2", ["bar2"], []), + ("/foo3", [], []), + ("/foo1/bar1", [], []), + ("/foo2/bar2", ["bar3"], []), + ("/foo2/bar2/bar3", [], ["test.txt"]), + ] + self.assertEqual(_walk, expected) + + def test_walk_filter_mix(self): + _walk = self._walk_step_names(filter_glob=["/foo2/bar**"], filter=["top1.txt"]) + expected = [ + ("/", ["foo2"], []), + ("/foo2", ["bar2"], []), + ("/foo2/bar2", ["bar3"], []), + ("/foo2/bar2/bar3", [], []), + ] + self.assertEqual(_walk, expected) + + def test_walk_exclude_dirs(self): + _walk = self._walk_step_names(exclude_dirs=["bar*", "foo2"]) + expected = [ + ("/", ["foo1", "foo3"], []), + ("/foo1", [], ["top1.txt", "top2.txt"]), + ("/foo3", [], []), + ] + self.assertEqual(_walk, expected) + + def test_walk_exclude_glob(self): + _walk = self._walk_step_names(exclude_glob=["**/top*", "test.txt"]) + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ("/foo1", ["bar1"], []), + ("/foo2", ["bar2"], []), + ("/foo3", [], []), + ("/foo1/bar1", [], []), + ("/foo2/bar2", ["bar3"], []), + ("/foo2/bar2/bar3", [], ["test.txt"]), + ] + self.assertEqual(_walk, expected) + def test_walk_depth(self): - _walk = [] - for step in self.fs.walk(search="depth"): - self.assertIsInstance(step, walk.Step) - path, dirs, files = step - _walk.append( - (path, [info.name for info in dirs], [info.name for info in files]) - ) + _walk = self._walk_step_names(search="depth") expected = [ ("/foo1/bar1", [], []), ("/foo1", ["bar1"], ["top1.txt", "top2.txt"]), @@ -98,14 +217,8 @@ def test_walk_depth(self): ] self.assertEqual(_walk, expected) - def test_walk_directory(self): - _walk = [] - for step in self.fs.walk("foo2"): - self.assertIsInstance(step, walk.Step) - path, dirs, files = step - _walk.append( - (path, [info.name for info in dirs], [info.name for info in files]) - ) + def test_walk_path(self): + _walk = self._walk_step_names("foo2") expected = [ ("/foo2", ["bar2"], ["top3.bin"]), ("/foo2/bar2", ["bar3"], []), @@ -113,34 +226,22 @@ def test_walk_directory(self): ] self.assertEqual(_walk, expected) - def test_walk_levels_1(self): - results = list(self.fs.walk(max_depth=1)) - self.assertEqual(len(results), 1) - dirs = sorted(info.name for info in results[0].dirs) - self.assertEqual(dirs, ["foo1", "foo2", "foo3"]) - files = sorted(info.name for info in results[0].files) - self.assertEqual(files, []) + def test_walk_max_depth_1_breadth(self): + _walk = self._walk_step_names(max_depth=1, search="breadth") + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ] + self.assertEqual(_walk, expected) - def test_walk_levels_1_depth(self): - results = list(self.fs.walk(max_depth=1, search="depth")) - self.assertEqual(len(results), 1) - dirs = sorted(info.name for info in results[0].dirs) - self.assertEqual(dirs, ["foo1", "foo2", "foo3"]) - files = sorted(info.name for info in results[0].files) - self.assertEqual(files, []) + def test_walk_max_depth_1_depth(self): + _walk = self._walk_step_names(max_depth=1, search="depth") + expected = [ + ("/", ["foo1", "foo2", "foo3"], []), + ] + self.assertEqual(_walk, expected) - def test_walk_levels_2(self): - _walk = [] - for step in self.fs.walk(max_depth=2): - self.assertIsInstance(step, walk.Step) - path, dirs, files = step - _walk.append( - ( - path, - sorted(info.name for info in dirs), - sorted(info.name for info in files), - ) - ) + def test_walk_max_depth_2(self): + _walk = self._walk_step_names(max_depth=2) expected = [ ("/", ["foo1", "foo2", "foo3"], []), ("/foo1", ["bar1"], ["top1.txt", "top2.txt"]), @@ -149,6 +250,30 @@ def test_walk_levels_2(self): ] self.assertEqual(_walk, expected) + +class TestDirs(TestBoundWalkerBase): + def test_walk_dirs(self): + dirs = list(self.fs.walk.dirs()) + self.assertEqual( + dirs, + ["/foo1", "/foo2", "/foo3", "/foo1/bar1", "/foo2/bar2", "/foo2/bar2/bar3"], + ) + + dirs = list(self.fs.walk.dirs(search="depth")) + self.assertEqual( + dirs, + ["/foo1/bar1", "/foo1", "/foo2/bar2/bar3", "/foo2/bar2", "/foo2", "/foo3"], + ) + + dirs = list(self.fs.walk.dirs(search="depth", exclude_dirs=["foo2"])) + self.assertEqual(dirs, ["/foo1/bar1", "/foo1", "/foo3"]) + + def test_foo(self): + dirs = list(self.fs.walk.dirs(search="depth", exclude_dirs=["foo2"])) + self.assertEqual(dirs, ["/foo1/bar1", "/foo1", "/foo3"]) + + +class TestFiles(TestBoundWalkerBase): def test_walk_files(self): files = list(self.fs.walk.files()) @@ -173,22 +298,6 @@ def test_walk_files(self): ], ) - def test_walk_dirs(self): - dirs = list(self.fs.walk.dirs()) - self.assertEqual( - dirs, - ["/foo1", "/foo2", "/foo3", "/foo1/bar1", "/foo2/bar2", "/foo2/bar2/bar3"], - ) - - dirs = list(self.fs.walk.dirs(search="depth")) - self.assertEqual( - dirs, - ["/foo1/bar1", "/foo1", "/foo2/bar2/bar3", "/foo2/bar2", "/foo2", "/foo3"], - ) - - dirs = list(self.fs.walk.dirs(search="depth", exclude_dirs=["foo2"])) - self.assertEqual(dirs, ["/foo1/bar1", "/foo1", "/foo3"]) - def test_walk_files_filter(self): files = list(self.fs.walk.files(filter=["*.txt"])) @@ -209,6 +318,16 @@ def test_walk_files_filter(self): self.assertEqual(files, []) + def test_walk_files_filter_glob(self): + files = list(self.fs.walk.files(filter_glob=["/foo2/**"])) + self.assertEqual( + files, + [ + "/foo2/top3.bin", + "/foo2/bar2/bar3/test.txt", + ], + ) + def test_walk_files_exclude(self): # Test exclude argument works files = list(self.fs.walk.files(exclude=["*.txt"])) @@ -222,25 +341,6 @@ def test_walk_files_exclude(self): files = list(self.fs.walk.files(exclude=["*"])) self.assertEqual(files, []) - def test_walk_info(self): - walk = [] - for path, info in self.fs.walk.info(): - walk.append((path, info.is_dir, info.name)) - - expected = [ - ("/foo1", True, "foo1"), - ("/foo2", True, "foo2"), - ("/foo3", True, "foo3"), - ("/foo1/top1.txt", False, "top1.txt"), - ("/foo1/top2.txt", False, "top2.txt"), - ("/foo1/bar1", True, "bar1"), - ("/foo2/bar2", True, "bar2"), - ("/foo2/top3.bin", False, "top3.bin"), - ("/foo2/bar2/bar3", True, "bar3"), - ("/foo2/bar2/bar3/test.txt", False, "test.txt"), - ] - self.assertEqual(walk, expected) - def test_broken(self): original_scandir = self.fs.scandir @@ -257,10 +357,6 @@ def broken_scandir(path, namespaces=None): with self.assertRaises(FSError): list(self.fs.walk.files(on_error=lambda path, error: False)) - def test_on_error_invalid(self): - with self.assertRaises(TypeError): - walk.Walker(on_error="nope") - def test_subdir_uses_same_walker(self): class CustomWalker(walk.Walker): @classmethod @@ -284,19 +380,32 @@ class CustomizedMemoryFS(MemoryFS): self.assertEqual(sub_walker.walker_class, CustomWalker) six.assertCountEqual(self, ["/c", "/d"], sub_walker.files()) - def test_readonly_wrapper_uses_same_walker(self): + def test_check_file_overwrite(self): class CustomWalker(walk.Walker): - @classmethod - def bind(cls, fs): - return walk.BoundWalker(fs, walker_class=CustomWalker) + def check_file(self, fs, info): + return False - class CustomizedMemoryFS(MemoryFS): - walker_class = CustomWalker + walker = CustomWalker() + files = list(walker.files(self.fs)) + self.assertEqual(files, []) - base_fs = CustomizedMemoryFS() - base_walker = base_fs.walk - self.assertEqual(base_walker.walker_class, CustomWalker) - readonly_fs = read_only(CustomizedMemoryFS()) - readonly_walker = readonly_fs.walk - self.assertEqual(readonly_walker.walker_class, CustomWalker) +class TestInfo(TestBoundWalkerBase): + def test_walk_info(self): + walk = [] + for path, info in self.fs.walk.info(): + walk.append((path, info.is_dir, info.name)) + + expected = [ + ("/foo1", True, "foo1"), + ("/foo2", True, "foo2"), + ("/foo3", True, "foo3"), + ("/foo1/top1.txt", False, "top1.txt"), + ("/foo1/top2.txt", False, "top2.txt"), + ("/foo1/bar1", True, "bar1"), + ("/foo2/bar2", True, "bar2"), + ("/foo2/top3.bin", False, "top3.bin"), + ("/foo2/bar2/bar3", True, "bar3"), + ("/foo2/bar2/bar3/test.txt", False, "test.txt"), + ] + self.assertEqual(walk, expected) From d85d9dbddf826d856242fd2c8cb3724c68d18f93 Mon Sep 17 00:00:00 2001 From: atollk Date: Sun, 5 Sep 2021 13:00:30 +0200 Subject: [PATCH 282/309] Changes from code review. --- CHANGELOG.md | 2 +- fs/base.py | 8 ++-- fs/glob.py | 95 +++++++++++++++++++++++++--------------------- fs/walk.py | 8 ++-- tests/test_glob.py | 61 ++++++++++++++++++++++++++--- 5 files changed, 116 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c370c9d..837f4947 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added -- To `fs.walk.Walker`, added parameters `filter_glob` and `exclude_glob`. +- Added `filter_glob` and `exclude_glob` parameters to `fs.walk.Walker`. Closes [#459](https://github.com/PyFilesystem/pyfilesystem2/issues/459). ### Fixed diff --git a/fs/base.py b/fs/base.py index dfbc6106..56e5cf99 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1653,8 +1653,8 @@ def check(self): if self.isclosed(): raise errors.FilesystemClosed() - def match(self, patterns, name, accept_prefix=False): - # type: (Optional[Iterable[Text]], Text, bool) -> bool + def match(self, patterns, name): + # type: (Optional[Iterable[Text]], Text) -> bool """Check if a name matches any of a list of wildcards. If a filesystem is case *insensitive* (such as Windows) then @@ -1711,7 +1711,7 @@ def match_glob(self, patterns, path, accept_prefix=False): ``['*.py']``, or `None` to match everything. path (str): A resource path, starting with "/". accept_prefix (bool): If ``True``, the path is - not required to match the wildcards themselves + not required to match the patterns themselves but only need to be a prefix of a string that does. Returns: @@ -1729,6 +1729,8 @@ def match_glob(self, patterns, path, accept_prefix=False): False >>> my_fs.match_glob(['dir/file.txt'], '/dir/', accept_prefix=True) True + >>> my_fs.match_glob(['dir/file.txt'], '/dir/', accept_prefix=False) + False >>> my_fs.match_glob(['dir/file.txt'], '/dir/gile.txt', accept_prefix=True) False diff --git a/fs/glob.py b/fs/glob.py index 82eb0228..4e783652 100644 --- a/fs/glob.py +++ b/fs/glob.py @@ -34,10 +34,10 @@ _PATTERN_CACHE = LRUCache( 1000 -) # type: LRUCache[Tuple[Text, bool], Tuple[int, bool, Pattern]] +) # type: LRUCache[Tuple[Text, bool], Tuple[Optional[int], Pattern]] -def _split_pattern_by_rec(pattern): +def _split_pattern_by_sep(pattern): # type: (Text) -> List[Text] """Split a glob pattern at its directory seperators (/). @@ -57,28 +57,27 @@ def _split_pattern_by_rec(pattern): return [pattern[i + 1 : j] for i, j in zip(indices[:-1], indices[1:])] -def _translate(pattern, case_sensitive=True): - # type: (Text, bool) -> Text - """Translate a wildcard pattern to a regular expression. +def _translate(pattern): + # type: (Text) -> Text + """Translate a glob pattern without '**' to a regular expression. There is no way to quote meta-characters. + Arguments: - pattern (str): A wildcard pattern. - case_sensitive (bool): Set to `False` to use a case - insensitive regex (default `True`). + pattern (str): A glob pattern. Returns: str: A regex equivalent to the given pattern. """ - if not case_sensitive: - pattern = pattern.lower() i, n = 0, len(pattern) res = [] while i < n: c = pattern[i] i = i + 1 if c == "*": + if i < n and pattern[i] == "*": + raise ValueError("glob._translate does not support '**' patterns.") res.append("[^/]*") elif c == "?": res.append("[^/]") @@ -96,7 +95,7 @@ def _translate(pattern, case_sensitive=True): stuff = pattern[i:j].replace("\\", "\\\\") i = j + 1 if stuff[0] == "!": - stuff = "^" + stuff[1:] + stuff = "^/" + stuff[1:] elif stuff[0] == "^": stuff = "\\" + stuff res.append("[%s]" % stuff) @@ -105,27 +104,35 @@ def _translate(pattern, case_sensitive=True): return "".join(res) -def _translate_glob(pattern, case_sensitive=True): - levels = 0 +def _translate_glob(pattern): + # type: (Text) -> Tuple[Optional[int], Text] + """Translate a glob pattern to a regular expression. + + There is no way to quote meta-characters. + + Arguments: + pattern (str): A glob pattern. + + Returns: + Tuple[Optional[int], Text]: The first component describes the levels + of depth this glob pattern goes to; basically the number of "/" in + the pattern. If there is a "**" in the glob pattern, the depth is + basically unbounded, and this component is `None` instead. + The second component is the regular expression. + + """ recursive = False re_patterns = [""] for component in iteratepath(pattern): if "**" in component: recursive = True split = component.split("**") - split_re = [_translate(s, case_sensitive=case_sensitive) for s in split] + split_re = [_translate(s) for s in split] re_patterns.append("/?" + ".*/?".join(split_re)) else: - re_patterns.append( - "/" + _translate(component, case_sensitive=case_sensitive) - ) - levels += 1 + re_patterns.append("/" + _translate(component)) re_glob = "(?ms)^" + "".join(re_patterns) + ("/$" if pattern.endswith("/") else "$") - return ( - levels, - recursive, - re.compile(re_glob, 0 if case_sensitive else re.IGNORECASE), - ) + return pattern.count("/") + 1 if not recursive else None, re_glob def match(pattern, path): @@ -147,10 +154,11 @@ def match(pattern, path): """ try: - levels, recursive, re_pattern = _PATTERN_CACHE[(pattern, True)] + levels, re_pattern = _PATTERN_CACHE[(pattern, True)] except KeyError: - levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True) - _PATTERN_CACHE[(pattern, True)] = (levels, recursive, re_pattern) + levels, re_str = _translate_glob(pattern) + re_pattern = re.compile(re_str) + _PATTERN_CACHE[(pattern, True)] = (levels, re_pattern) if path and path[0] != "/": path = "/" + path return bool(re_pattern.match(path)) @@ -169,10 +177,11 @@ def imatch(pattern, path): """ try: - levels, recursive, re_pattern = _PATTERN_CACHE[(pattern, False)] + levels, re_pattern = _PATTERN_CACHE[(pattern, False)] except KeyError: - levels, recursive, re_pattern = _translate_glob(pattern, case_sensitive=True) - _PATTERN_CACHE[(pattern, False)] = (levels, recursive, re_pattern) + levels, re_str = _translate_glob(pattern) + re_pattern = re.compile(re_str, re.IGNORECASE) + _PATTERN_CACHE[(pattern, False)] = (levels, re_pattern) if path and path[0] != "/": path = "/" + path return bool(re_pattern.match(path)) @@ -187,7 +196,7 @@ def match_any(patterns, path): Arguments: patterns (list): A list of wildcard pattern, e.g ``["*.py", "*.pyc"]`` - name (str): A filename. + path (str): A resource path. Returns: bool: `True` if the path matches at least one of the patterns. @@ -207,7 +216,7 @@ def imatch_any(patterns, path): Arguments: patterns (list): A list of wildcard pattern, e.g ``["*.py", "*.pyc"]`` - name (str): A filename. + path (str): A resource path. Returns: bool: `True` if the path matches at least one of the patterns. @@ -228,16 +237,17 @@ def get_matcher(patterns, case_sensitive, accept_prefix=False): case_sensitive (bool): If ``True``, then the callable will be case sensitive, otherwise it will be case insensitive. accept_prefix (bool): If ``True``, the name is - not required to match the wildcards themselves + not required to match the patterns themselves but only need to be a prefix of a string that does. Returns: callable: a matcher that will return `True` if the paths given as - an argument matches any of the given patterns. + an argument matches any of the given patterns, or if no patterns + exist. Example: - >>> from fs import wildcard - >>> is_python = wildcard.get_matcher(['*.py'], True) + >>> from fs import glob + >>> is_python = glob.get_matcher(['*.py'], True) >>> is_python('__init__.py') True >>> is_python('foo.txt') @@ -245,12 +255,12 @@ def get_matcher(patterns, case_sensitive, accept_prefix=False): """ if not patterns: - return lambda name: True + return lambda path: True if accept_prefix: new_patterns = [] for pattern in patterns: - split = _split_pattern_by_rec(pattern) + split = _split_pattern_by_sep(pattern) for i in range(1, len(split)): new_pattern = "/".join(split[:i]) new_patterns.append(new_pattern) @@ -310,18 +320,15 @@ def __repr__(self): def _make_iter(self, search="breadth", namespaces=None): # type: (str, List[str]) -> Iterator[GlobMatch] try: - levels, recursive, re_pattern = _PATTERN_CACHE[ - (self.pattern, self.case_sensitive) - ] + levels, re_pattern = _PATTERN_CACHE[(self.pattern, self.case_sensitive)] except KeyError: - levels, recursive, re_pattern = _translate_glob( - self.pattern, case_sensitive=self.case_sensitive - ) + levels, re_str = _translate_glob(self.pattern) + re_pattern = re.compile(re_str, 0 if self.case_sensitive else re.IGNORECASE) for path, info in self.fs.walk.info( path=self.path, namespaces=namespaces or self.namespaces, - max_depth=None if recursive else levels, + max_depth=levels, search=search, exclude_dirs=self.exclude_dirs, ): diff --git a/fs/walk.py b/fs/walk.py index 1724dfcc..b743e6f2 100644 --- a/fs/walk.py +++ b/fs/walk.py @@ -84,13 +84,13 @@ def __init__( a list of filename patterns, e.g. ``["~*"]``. Files matching any of these patterns will be removed from the walk. filter_dirs (list, optional): A list of patterns that will be used - to match directories paths. The walk will only open directories + to match directories names. The walk will only open directories that match at least one of these patterns. Directories will only be returned if the final component matches one of the patterns. exclude_dirs (list, optional): A list of patterns that will be used to filter out directories from the walk. e.g. - ``['*.svn', '*.git']``. Directories matching any of these + ``['*.svn', '*.git']``. Directory names matching any of these patterns will be removed from the walk. max_depth (int, optional): Maximum directory depth to walk. filter_glob (list, optional): If supplied, this parameter @@ -98,7 +98,7 @@ def __init__( Resources will only be returned if their global path or an extension of it matches one of the patterns. exclude_glob (list, optional): If supplied, this parameter - should be a list of path patterns e.g. ``["foo/**/*.py"]``. + should be a list of path patterns e.g. ``["foo/**/*.pyc"]``. Resources will not be returned if their global path or an extension of it matches one of the patterns. @@ -213,7 +213,7 @@ def _iter_walk( def _check_open_dir(self, fs, path, info): # type: (FS, Text, Info) -> bool """Check if a directory should be considered in the walk.""" - full_path = ("" if path == "/" else path) + "/" + info.name + full_path = combine(path, info.name) if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name): return False if self.exclude_glob is not None and fs.match_glob( diff --git a/tests/test_glob.py b/tests/test_glob.py index 6147bf49..9a5d8827 100644 --- a/tests/test_glob.py +++ b/tests/test_glob.py @@ -1,7 +1,10 @@ from __future__ import unicode_literals +import re import unittest +from parameterized import parameterized + from fs import glob, open_fs @@ -17,8 +20,8 @@ def setUp(self): fs.makedirs("a/b/c/").writetext("foo.py", "import fs") repr(fs.glob) - def test_match(self): - tests = [ + @parameterized.expand( + [ ("*.?y", "/test.py", True), ("*.py", "/test.py", True), ("*.py", "__init__.py", True), @@ -41,11 +44,11 @@ def test_match(self): ("**/", "/test/", True), ("**/", "/test.py", False), ] - for pattern, path, expected in tests: - self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) + ) + def test_match(self, pattern, path, expected): + self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) # Run a second time to test cache - for pattern, path, expected in tests: - self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) + self.assertEqual(glob.match(pattern, path), expected, msg=(pattern, path)) def test_count_1dir(self): globber = glob.BoundGlobber(self.fs) @@ -99,3 +102,49 @@ def test_remove_all(self): globber = glob.BoundGlobber(self.fs) globber("**").remove() self.assertEqual(sorted(self.fs.listdir("/")), []) + + translate_test_cases = [ + ("foo.py", ["foo.py"], ["Foo.py", "foo_py", "foo", ".py"]), + ("foo?py", ["foo.py", "fooapy"], ["foo/py", "foopy", "fopy"]), + ("bar/foo.py", ["bar/foo.py"], []), + ("bar?foo.py", ["barafoo.py"], ["bar/foo.py"]), + ("???.py", ["foo.py", "bar.py", "FOO.py"], [".py", "foo.PY"]), + ("bar/*.py", ["bar/.py", "bar/foo.py"], ["bar/foo"]), + ("bar/foo*.py", ["bar/foo.py", "bar/foobaz.py"], ["bar/foo", "bar/.py"]), + ("*/[bar]/foo.py", ["/b/foo.py", "x/a/foo.py", "/r/foo.py"], ["b/foo.py", "/bar/foo.py"]), + ("[!bar]/foo.py", ["x/foo.py"], ["//foo.py"]), + ("[.py", ["[.py"], [".py", "."]), + ] + + @parameterized.expand(translate_test_cases) + def test_translate(self, glob_pattern, expected_matches, expected_not_matches): + translated = glob._translate(glob_pattern) + for m in expected_matches: + self.assertTrue(re.match(translated, m)) + for m in expected_not_matches: + self.assertFalse(re.match(translated, m)) + + @parameterized.expand(translate_test_cases) + def test_translate_glob_simple(self, glob_pattern, expected_matches, expected_not_matches): + levels, translated = glob._translate_glob(glob_pattern) + self.assertEqual(levels, glob_pattern.count("/") + 1) + for m in expected_matches: + self.assertTrue(re.match(translated, "/" + m)) + for m in expected_not_matches: + self.assertFalse(re.match(translated, m)) + self.assertFalse(re.match(translated, "/" + m)) + + @parameterized.expand( + [ + ("foo/**/bar", ["/foo/bar", "/foo/baz/bar", "/foo/baz/qux/bar"], ["/foo"]), + ("**/*/bar", ["/foo/bar", "/foo/bar"], ["/bar", "/bar"]), + ("/**/foo/**/bar", ["/baz/foo/qux/bar", "/foo/bar"], ["/bar"]), + ] + ) + def test_translate_glob(self, glob_pattern, expected_matches, expected_not_matches): + levels, translated = glob._translate_glob(glob_pattern) + self.assertIsNone(levels) + for m in expected_matches: + self.assertTrue(re.match(translated, m)) + for m in expected_not_matches: + self.assertFalse(re.match(translated, m)) From cd545b2a8dee3f6677ec284ddad4bc09ae7e3f2b Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sat, 16 Jul 2022 07:50:14 +1000 Subject: [PATCH 283/309] Fx typo in `docs/source/info.rst` (#544) There is a small typo in docs/source/info.rst. Should read `filesystem` rather than `fileystem`. Signed-off-by: Tim Gates --- docs/source/info.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/info.rst b/docs/source/info.rst index ba0d6d12..82c3e076 100644 --- a/docs/source/info.rst +++ b/docs/source/info.rst @@ -46,7 +46,7 @@ file:: resource_info = fs.getinfo('myfile.txt', namespaces=['details', 'access']) -In addition to the specified namespaces, the fileystem will also return +In addition to the specified namespaces, the filesystem will also return the ``basic`` namespace, which contains the name of the resource, and a flag which indicates if the resource is a directory. From ec7c68d19cfee4bb037302911538989f5e9b08d5 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 11:26:29 +0200 Subject: [PATCH 284/309] add failing tests --- fs/test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/fs/test.py b/fs/test.py index 232666da..c15658ec 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1811,6 +1811,28 @@ def test_move_file_mem(self): def test_move_file_temp(self): self._test_move_file("temp://") + def test_move_file_onto_itself(self): + self.fs.writetext("file.txt", "Hello") + self.fs.move("file.txt", "file.txt", overwrite=True) + self.assert_text("file.txt", "Hello") + + def test_move_file_onto_itself_relpath(self): + subdir = self.fs.makedir("sub") + subdir.writetext("file.txt", "Hello") + self.fs.move("sub/file.txt", "sub/../sub/file.txt", overwrite=True) + self.assert_text("sub/file.txt", "Hello") + + def test_copy_file_onto_itself(self): + self.fs.writetext("file.txt", "Hello") + self.fs.copy("file.txt", "file.txt", overwrite=True) + self.assert_text("file.txt", "Hello") + + def test_copy_file_onto_itself_relpath(self): + subdir = self.fs.makedir("sub") + subdir.writetext("file.txt", "Hello") + self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=True) + self.assert_text("sub/file.txt", "Hello") + def test_copydir(self): self.fs.makedirs("foo/bar/baz/egg") self.fs.writetext("foo/bar/foofoo.txt", "Hello") From 56bed8364013ede9d10035ab7def5d35898163da Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 12:42:36 +0200 Subject: [PATCH 285/309] fix OSFS copy --- fs/osfs.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 0add3d97..dc4e852f 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -409,7 +409,7 @@ def _check_copy(self, src_path, dst_path, overwrite=False): if self.gettype(src_path) is not ResourceType.file: raise errors.FileExpected(src_path) # check dst_path does not exist if we are not overwriting - if not overwrite and self.exists(_dst_path): + if not overwrite and _src_path != _dst_path and self.exists(_dst_path): raise errors.DestinationExists(dst_path) # check parent dir of _dst_path exists and is a directory if self.gettype(dirname(dst_path)) is not ResourceType.directory: @@ -440,6 +440,9 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): self.getsyspath(_src_path), self.getsyspath(_dst_path), ) + # exit early if we copy the file onto itself + if overwrite and _src_sys == _dst_sys: + return # attempt using sendfile try: # initialise variables to pass to sendfile @@ -467,7 +470,14 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None with self._lock: _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) - shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path)) + _src_sys, _dst_sys = ( + self.getsyspath(_src_path), + self.getsyspath(_dst_path), + ) + # exit early if we copy the file onto itself + if overwrite and _src_sys == _dst_sys: + return + shutil.copy2(_src_sys, _dst_sys) # --- Backport of os.scandir for Python < 3.5 ------------ From 0e3ed7214234f8fc7caca4bcdc13db7e78d231b2 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 12:53:05 +0200 Subject: [PATCH 286/309] assert DestinationExists when using overwrite=False --- fs/base.py | 3 +++ fs/osfs.py | 2 +- fs/test.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/fs/base.py b/fs/base.py index 56e5cf99..5d100676 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1161,6 +1161,9 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): raise errors.DestinationExists(dst_path) if self.getinfo(src_path).is_dir: raise errors.FileExpected(src_path) + if normpath(src_path) == normpath(dst_path): + # early exit when moving a file onto itself + return if self.getmeta().get("supports_rename", False): try: src_sys_path = self.getsyspath(src_path) diff --git a/fs/osfs.py b/fs/osfs.py index dc4e852f..ed6b0953 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -409,7 +409,7 @@ def _check_copy(self, src_path, dst_path, overwrite=False): if self.gettype(src_path) is not ResourceType.file: raise errors.FileExpected(src_path) # check dst_path does not exist if we are not overwriting - if not overwrite and _src_path != _dst_path and self.exists(_dst_path): + if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) # check parent dir of _dst_path exists and is a directory if self.gettype(dirname(dst_path)) is not ResourceType.directory: diff --git a/fs/test.py b/fs/test.py index c15658ec..108239b9 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1816,23 +1816,35 @@ def test_move_file_onto_itself(self): self.fs.move("file.txt", "file.txt", overwrite=True) self.assert_text("file.txt", "Hello") + with self.assertRaises(errors.DestinationExists): + self.fs.move("file.txt", "file.txt", overwrite=False) + def test_move_file_onto_itself_relpath(self): subdir = self.fs.makedir("sub") subdir.writetext("file.txt", "Hello") self.fs.move("sub/file.txt", "sub/../sub/file.txt", overwrite=True) self.assert_text("sub/file.txt", "Hello") + with self.assertRaises(errors.DestinationExists): + self.fs.move("sub/file.txt", "sub/../sub/file.txt", overwrite=False) + def test_copy_file_onto_itself(self): self.fs.writetext("file.txt", "Hello") self.fs.copy("file.txt", "file.txt", overwrite=True) self.assert_text("file.txt", "Hello") + with self.assertRaises(errors.DestinationExists): + self.fs.copy("file.txt", "file.txt", overwrite=False) + def test_copy_file_onto_itself_relpath(self): subdir = self.fs.makedir("sub") subdir.writetext("file.txt", "Hello") self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=True) self.assert_text("sub/file.txt", "Hello") + with self.assertRaises(errors.DestinationExists): + self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=False) + def test_copydir(self): self.fs.makedirs("foo/bar/baz/egg") self.fs.writetext("foo/bar/foofoo.txt", "Hello") From b3d2d733302c08bc81771959268a0bc0a436a805 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 13:08:37 +0200 Subject: [PATCH 287/309] fix memoryfs --- fs/base.py | 13 +++++++++---- fs/memoryfs.py | 6 ++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/fs/base.py b/fs/base.py index 5d100676..6ef1b5bd 100644 --- a/fs/base.py +++ b/fs/base.py @@ -423,13 +423,18 @@ def copy( """ with self._lock: - if not overwrite and self.exists(dst_path): + _src_path = self.validatepath(src_path) + _dst_path = self.validatepath(dst_path) + if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) - with closing(self.open(src_path, "rb")) as read_file: + if overwrite and _src_path == _dst_path: + # exit early when copying a file onto itself + return + with closing(self.open(_src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO - self.upload(dst_path, read_file) # type: ignore + self.upload(_dst_path, read_file) # type: ignore if preserve_time: - copy_modified_time(self, src_path, self, dst_path) + copy_modified_time(self, _src_path, self, _dst_path) def copydir( self, diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 8efdc139..8c428d35 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -462,6 +462,12 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): elif not overwrite and dst_name in dst_dir_entry: raise errors.DestinationExists(dst_path) + # handle moving a file onto itself + if src_dir == dst_dir and src_name == dst_name: + if overwrite: + return + raise errors.DestinationExists(dst_path) + # move the entry from the src folder to the dst folder dst_dir_entry.set_entry(dst_name, src_entry) src_dir_entry.remove_entry(src_name) From f1a7bfb85cc9bd75f68078591990b6f7ca9e9882 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 13:17:50 +0200 Subject: [PATCH 288/309] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837f4947..4ffe9392 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - Elaborated documentation of `filter_dirs` and `exclude_dirs` in `fs.walk.Walker`. Closes [#371](https://github.com/PyFilesystem/pyfilesystem2/issues/371). - +- Fixed a bug where files could be truncated or deleted when moved / copied onto itself. + Closes [#546](https://github.com/PyFilesystem/pyfilesystem2/issues/546) ## [2.4.16] - 2022-05-02 From cec3e06611feda7ee40cc6dce8544f43ec0f01a4 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 17:13:37 +0200 Subject: [PATCH 289/309] add movedir tests --- fs/errors.py | 7 +++++++ fs/test.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/fs/errors.py b/fs/errors.py index 32c795d9..d986f179 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -34,6 +34,7 @@ "IllegalBackReference", "InsufficientStorage", "InvalidCharsInPath", + "InvalidMoveOperation", "InvalidPath", "MissingInfoNamespace", "NoSysPath", @@ -303,6 +304,12 @@ class DirectoryNotEmpty(ResourceError): default_message = "directory '{path}' is not empty" +class InvalidMoveOperation(ResourceError): + """Attempt to move a folder into its own subfolder.""" + + default_message = "you cannot move '{path}' into its own subfolder" + + class ResourceLocked(ResourceError): """Attempt to use a locked resource.""" diff --git a/fs/test.py b/fs/test.py index 108239b9..a4e27d9c 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1885,6 +1885,25 @@ def test_movedir(self): with self.assertRaises(errors.DirectoryExpected): self.fs.movedir("foo2/foofoo.txt", "foo2/baz/egg") + def test_movedir_onto_itself(self): + folder = self.fs.makedir("folder") + folder.writetext("file1.txt", "Hello1") + sub = folder.makedir("sub") + sub.writetext("file2.txt", "Hello2") + + self.fs.movedir("folder", "folder") + self.assert_text("folder/file1.txt", "Hello1") + self.assert_text("folder/sub/file2.txt", "Hello2") + + def test_movedir_into_its_own_subfolder(self): + folder = self.fs.makedir("folder") + folder.writetext("file1.txt", "Hello1") + sub = folder.makedir("sub") + sub.writetext("file2.txt", "Hello2") + + with self.assertRaises(errors.InvalidMoveOperation): + self.fs.movedir("folder", "folder/sub/") + def test_match(self): self.assertTrue(self.fs.match(["*.py"], "foo.py")) self.assertEqual( From 2b0cbac410faf482095db83f5e86aa977dc79ae3 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 17:20:28 +0200 Subject: [PATCH 290/309] add copydir tests --- fs/test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/fs/test.py b/fs/test.py index a4e27d9c..870e4171 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1862,6 +1862,29 @@ def test_copydir(self): with self.assertRaises(errors.DirectoryExpected): self.fs.copydir("foo2/foofoo.txt", "foofoo.txt", create=True) + def test_copydir_onto_itself(self): + folder = self.fs.makedir("folder") + folder.writetext("file1.txt", "Hello1") + sub = folder.makedir("sub") + sub.writetext("file2.txt", "Hello2") + + self.fs.copydir("folder", "folder") + self.assert_text("folder/file1.txt", "Hello1") + self.assert_text("folder/sub/file2.txt", "Hello2") + + def test_copydir_into_its_own_subfolder(self): + # TODO: This test hangs forever at the moment. + # + # folder = self.fs.makedir("folder") + # folder.writetext("file1.txt", "Hello1") + # sub = folder.makedir("sub") + # sub.writetext("file2.txt", "Hello2") + # self.fs.copydir("folder", "folder/sub/") + # self.assert_text("folder/file1.txt", "Hello1") + # self.assert_text("folder/sub/file1.txt", "Hello1") + # self.assert_not_exists("folder/sub/file2.txt") + pass + def test_movedir(self): self.fs.makedirs("foo/bar/baz/egg") self.fs.writetext("foo/bar/foofoo.txt", "Hello") From 9a7fe1ad21b92d7cfa9d83c9457e0159d129ab28 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Fri, 22 Jul 2022 17:34:44 +0200 Subject: [PATCH 291/309] new error names --- fs/errors.py | 18 ++++++++++++------ fs/test.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/fs/errors.py b/fs/errors.py index d986f179..56c392db 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -241,6 +241,18 @@ class RemoveRootError(OperationFailed): default_message = "root directory may not be removed" +class IllegalMoveDestination(OperationFailed): + """Attempt to move a folder into its own subfolder.""" + + default_message = "'{path}' cannot be moved into itself" + + +class IllegalCopyDestination(OperationFailed): + """Attempt to copy a folder into its own subfolder.""" + + default_message = "'{path}' cannot be copied into itself" + + class ResourceError(FSError): """Base exception class for error associated with a specific resource.""" @@ -304,12 +316,6 @@ class DirectoryNotEmpty(ResourceError): default_message = "directory '{path}' is not empty" -class InvalidMoveOperation(ResourceError): - """Attempt to move a folder into its own subfolder.""" - - default_message = "you cannot move '{path}' into its own subfolder" - - class ResourceLocked(ResourceError): """Attempt to use a locked resource.""" diff --git a/fs/test.py b/fs/test.py index 870e4171..b68396d6 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1924,7 +1924,7 @@ def test_movedir_into_its_own_subfolder(self): sub = folder.makedir("sub") sub.writetext("file2.txt", "Hello2") - with self.assertRaises(errors.InvalidMoveOperation): + with self.assertRaises(errors.IllegalMoveDestination): self.fs.movedir("folder", "folder/sub/") def test_match(self): From 2a9330456cbafb2cbf74bd62f15f7d5939f2b156 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:05:48 +0200 Subject: [PATCH 292/309] establish new errors.IllegalDestination --- fs/base.py | 15 ++++++++++----- fs/errors.py | 16 +++++++--------- fs/memoryfs.py | 15 ++++++++++++--- fs/test.py | 29 +++++++++++++---------------- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/fs/base.py b/fs/base.py index 6ef1b5bd..a1cfdef0 100644 --- a/fs/base.py +++ b/fs/base.py @@ -21,11 +21,11 @@ from contextlib import closing from functools import partial, wraps -from . import copy, errors, fsencode, iotools, tools, walk, wildcard, glob +from . import copy, errors, fsencode, glob, iotools, tools, walk, wildcard from .copy import copy_modified_time from .glob import BoundGlobber from .mode import validate_open_mode -from .path import abspath, join, normpath +from .path import abspath, isbase, join, normpath from .time import datetime_to_epoch from .walk import Walker @@ -425,11 +425,10 @@ def copy( with self._lock: _src_path = self.validatepath(src_path) _dst_path = self.validatepath(dst_path) + if _src_path == _dst_path: + raise errors.IllegalDestination(dst_path) if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) - if overwrite and _src_path == _dst_path: - # exit early when copying a file onto itself - return with closing(self.open(_src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(_dst_path, read_file) # type: ignore @@ -1093,6 +1092,12 @@ def movedir(self, src_path, dst_path, create=False, preserve_time=False): from .move import move_dir with self._lock: + _src_path = self.validatepath(src_path) + _dst_path = self.validatepath(dst_path) + if _src_path == _dst_path: + return + if isbase(_src_path, _dst_path): + raise errors.IllegalDestination(dst_path) if not create and not self.exists(dst_path): raise errors.ResourceNotFound(dst_path) move_dir(self, src_path, self, dst_path, preserve_time=preserve_time) diff --git a/fs/errors.py b/fs/errors.py index 56c392db..adc9afa9 100644 --- a/fs/errors.py +++ b/fs/errors.py @@ -32,9 +32,9 @@ "FilesystemClosed", "FSError", "IllegalBackReference", + "IllegalDestination", "InsufficientStorage", "InvalidCharsInPath", - "InvalidMoveOperation", "InvalidPath", "MissingInfoNamespace", "NoSysPath", @@ -241,16 +241,14 @@ class RemoveRootError(OperationFailed): default_message = "root directory may not be removed" -class IllegalMoveDestination(OperationFailed): - """Attempt to move a folder into its own subfolder.""" +class IllegalDestination(OperationFailed): + """The given destination cannot be used for the operation. - default_message = "'{path}' cannot be moved into itself" - - -class IllegalCopyDestination(OperationFailed): - """Attempt to copy a folder into its own subfolder.""" + This error will occur when attempting to move / copy a folder into itself or copying + a file onto itself. + """ - default_message = "'{path}' cannot be copied into itself" + default_message = "'{path}' is not a legal destination" class ResourceError(FSError): diff --git a/fs/memoryfs.py b/fs/memoryfs.py index 8c428d35..0ca5ce16 100644 --- a/fs/memoryfs.py +++ b/fs/memoryfs.py @@ -19,7 +19,7 @@ from .enums import ResourceType, Seek from .info import Info from .mode import Mode -from .path import iteratepath, normpath, split +from .path import isbase, iteratepath, normpath, split if typing.TYPE_CHECKING: from typing import ( @@ -478,8 +478,17 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): copy_modified_time(self, src_path, self, dst_path) def movedir(self, src_path, dst_path, create=False, preserve_time=False): - src_dir, src_name = split(self.validatepath(src_path)) - dst_dir, dst_name = split(self.validatepath(dst_path)) + _src_path = self.validatepath(src_path) + _dst_path = self.validatepath(dst_path) + dst_dir, dst_name = split(_dst_path) + src_dir, src_name = split(_src_path) + + # move a dir onto itself + if _src_path == _dst_path: + return + # move a dir into itself + if isbase(_src_path, _dst_path): + raise errors.IllegalDestination(dst_path) with self._lock: src_dir_entry = self._get_dir_entry(src_dir) diff --git a/fs/test.py b/fs/test.py index b68396d6..715e6d20 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1830,20 +1830,20 @@ def test_move_file_onto_itself_relpath(self): def test_copy_file_onto_itself(self): self.fs.writetext("file.txt", "Hello") - self.fs.copy("file.txt", "file.txt", overwrite=True) - self.assert_text("file.txt", "Hello") - - with self.assertRaises(errors.DestinationExists): + with self.assertRaises(errors.IllegalDestination): + self.fs.copy("file.txt", "file.txt", overwrite=True) + with self.assertRaises(errors.IllegalDestination): self.fs.copy("file.txt", "file.txt", overwrite=False) + self.assert_text("file.txt", "Hello") def test_copy_file_onto_itself_relpath(self): subdir = self.fs.makedir("sub") subdir.writetext("file.txt", "Hello") - self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=True) - self.assert_text("sub/file.txt", "Hello") - - with self.assertRaises(errors.DestinationExists): + with self.assertRaises(errors.IllegalDestination): + self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=True) + with self.assertRaises(errors.IllegalDestination): self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=False) + self.assert_text("sub/file.txt", "Hello") def test_copydir(self): self.fs.makedirs("foo/bar/baz/egg") @@ -1868,21 +1868,18 @@ def test_copydir_onto_itself(self): sub = folder.makedir("sub") sub.writetext("file2.txt", "Hello2") - self.fs.copydir("folder", "folder") + with self.assertRaises(errors.IllegalDestination): + self.fs.copydir("folder", "folder") self.assert_text("folder/file1.txt", "Hello1") self.assert_text("folder/sub/file2.txt", "Hello2") def test_copydir_into_its_own_subfolder(self): - # TODO: This test hangs forever at the moment. - # # folder = self.fs.makedir("folder") # folder.writetext("file1.txt", "Hello1") # sub = folder.makedir("sub") # sub.writetext("file2.txt", "Hello2") - # self.fs.copydir("folder", "folder/sub/") - # self.assert_text("folder/file1.txt", "Hello1") - # self.assert_text("folder/sub/file1.txt", "Hello1") - # self.assert_not_exists("folder/sub/file2.txt") + # with self.assertRaises(errors.IllegalDestination): + # self.fs.copydir("folder", "folder/sub/") pass def test_movedir(self): @@ -1924,7 +1921,7 @@ def test_movedir_into_its_own_subfolder(self): sub = folder.makedir("sub") sub.writetext("file2.txt", "Hello2") - with self.assertRaises(errors.IllegalMoveDestination): + with self.assertRaises(errors.IllegalDestination): self.fs.movedir("folder", "folder/sub/") def test_match(self): From 0f097317b763b6deb975c506a21a69de7bd0a538 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:10:25 +0200 Subject: [PATCH 293/309] fix osfs --- fs/osfs.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index ed6b0953..e56bde03 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -408,6 +408,9 @@ def _check_copy(self, src_path, dst_path, overwrite=False): # check src_path exists and is a file if self.gettype(src_path) is not ResourceType.file: raise errors.FileExpected(src_path) + # it's not allowed to copy a file onto itself + if _src_path == _dst_path: + raise errors.IllegalDestination(dst_path) # check dst_path does not exist if we are not overwriting if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) @@ -440,9 +443,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): self.getsyspath(_src_path), self.getsyspath(_dst_path), ) - # exit early if we copy the file onto itself - if overwrite and _src_sys == _dst_sys: - return # attempt using sendfile try: # initialise variables to pass to sendfile @@ -474,9 +474,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): self.getsyspath(_src_path), self.getsyspath(_dst_path), ) - # exit early if we copy the file onto itself - if overwrite and _src_sys == _dst_sys: - return shutil.copy2(_src_sys, _dst_sys) # --- Backport of os.scandir for Python < 3.5 ------------ From 2e0238af8104d3b10d1e6a0abf5d1fff3583c4a7 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:12:23 +0200 Subject: [PATCH 294/309] fix copydir hangup --- fs/base.py | 4 ++++ fs/test.py | 13 ++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fs/base.py b/fs/base.py index a1cfdef0..a794bf48 100644 --- a/fs/base.py +++ b/fs/base.py @@ -461,6 +461,10 @@ def copydir( """ with self._lock: + _src_path = self.validatepath(src_path) + _dst_path = self.validatepath(dst_path) + if isbase(_src_path, _dst_path): + raise errors.IllegalDestination(dst_path) if not create and not self.exists(dst_path): raise errors.ResourceNotFound(dst_path) if not self.getinfo(src_path).is_dir: diff --git a/fs/test.py b/fs/test.py index 715e6d20..2b5bd343 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1874,13 +1874,12 @@ def test_copydir_onto_itself(self): self.assert_text("folder/sub/file2.txt", "Hello2") def test_copydir_into_its_own_subfolder(self): - # folder = self.fs.makedir("folder") - # folder.writetext("file1.txt", "Hello1") - # sub = folder.makedir("sub") - # sub.writetext("file2.txt", "Hello2") - # with self.assertRaises(errors.IllegalDestination): - # self.fs.copydir("folder", "folder/sub/") - pass + folder = self.fs.makedir("folder") + folder.writetext("file1.txt", "Hello1") + sub = folder.makedir("sub") + sub.writetext("file2.txt", "Hello2") + with self.assertRaises(errors.IllegalDestination): + self.fs.copydir("folder", "folder/sub/") def test_movedir(self): self.fs.makedirs("foo/bar/baz/egg") From e157d936cdd947c9b2508851508a436177a8a41d Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:16:10 +0200 Subject: [PATCH 295/309] uses validated paths in copydir --- fs/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/base.py b/fs/base.py index a794bf48..217cbb22 100644 --- a/fs/base.py +++ b/fs/base.py @@ -465,11 +465,11 @@ def copydir( _dst_path = self.validatepath(dst_path) if isbase(_src_path, _dst_path): raise errors.IllegalDestination(dst_path) - if not create and not self.exists(dst_path): + if not create and not self.exists(_dst_path): raise errors.ResourceNotFound(dst_path) - if not self.getinfo(src_path).is_dir: + if not self.getinfo(_src_path).is_dir: raise errors.DirectoryExpected(src_path) - copy.copy_dir(self, src_path, self, dst_path, preserve_time=preserve_time) + copy.copy_dir(self, _src_path, self, _dst_path, preserve_time=preserve_time) def create(self, path, wipe=False): # type: (Text, bool) -> bool From a039bb687519f1090bc19dd4b5e80c35891d53c1 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:16:36 +0200 Subject: [PATCH 296/309] assert no files were touched --- fs/test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/test.py b/fs/test.py index 2b5bd343..154eb32e 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1880,6 +1880,8 @@ def test_copydir_into_its_own_subfolder(self): sub.writetext("file2.txt", "Hello2") with self.assertRaises(errors.IllegalDestination): self.fs.copydir("folder", "folder/sub/") + self.assert_text("folder/file1.txt", "Hello1") + self.assert_text("folder/sub/file2.txt", "Hello2") def test_movedir(self): self.fs.makedirs("foo/bar/baz/egg") @@ -1922,6 +1924,8 @@ def test_movedir_into_its_own_subfolder(self): with self.assertRaises(errors.IllegalDestination): self.fs.movedir("folder", "folder/sub/") + self.assert_text("folder/file1.txt", "Hello1") + self.assert_text("folder/sub/file2.txt", "Hello2") def test_match(self): self.assertTrue(self.fs.match(["*.py"], "foo.py")) From f9cf49f03260322fb8672ee2d99be5d7e7862cbd Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Mon, 25 Jul 2022 11:43:19 +0200 Subject: [PATCH 297/309] fix wrapfs --- fs/wrapfs.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index abbbe4e3..2fbb7050 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -12,7 +12,7 @@ from .copy import copy_dir, copy_file from .error_tools import unwrap_errors from .info import Info -from .path import abspath, join, normpath +from .path import abspath, isbase, join, normpath if typing.TYPE_CHECKING: from typing import ( @@ -267,7 +267,11 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) + _val_src_path = self.validatepath(_src_path) + _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): + if src_fs == dst_fs and _val_src_path == _val_dst_path: + raise errors.IllegalDestination(_dst_path) if not overwrite and dst_fs.exists(_dst_path): raise errors.DestinationExists(_dst_path) copy_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) @@ -276,7 +280,11 @@ def copydir(self, src_path, dst_path, create=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) + _val_src_path = self.validatepath(_src_path) + _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): + if src_fs == dst_fs and isbase(_val_src_path, _val_dst_path): + raise errors.IllegalDestination(_dst_path) if not create and not dst_fs.exists(_dst_path): raise errors.ResourceNotFound(dst_path) if not src_fs.getinfo(_src_path).is_dir: From b4f2dc430efaebc1cb4773887bd0216cf5f04b3c Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 10:41:27 +0200 Subject: [PATCH 298/309] wip copy --- fs/copy.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 90848c76..ae3acd8b 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -7,9 +7,9 @@ import warnings -from .errors import ResourceNotFound +from .errors import IllegalDestination, ResourceNotFound from .opener import manage_fs -from .path import abspath, combine, frombase, normpath +from .path import abspath, combine, frombase, isbase, normpath from .tools import is_thread_safe from .walk import Walker @@ -438,6 +438,8 @@ def copy_dir_if( dst_fs, create=True ) as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): + if src_fs == dst_fs and isbase(_src_path, _dst_path): + raise IllegalDestination(dst_path) _thread_safe = is_thread_safe(_src_fs, _dst_fs) with Copier( num_workers=workers if _thread_safe else 0, preserve_time=preserve_time From 2668bfcb3141e2923a3be60c35c8bc70785f41c7 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 11:20:45 +0200 Subject: [PATCH 299/309] raise DestinationExists if overwrite is not set --- fs/base.py | 4 ++-- fs/copy.py | 10 ++++++---- fs/osfs.py | 6 +++--- fs/test.py | 4 ++-- fs/wrapfs.py | 2 -- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/fs/base.py b/fs/base.py index 217cbb22..189d45ab 100644 --- a/fs/base.py +++ b/fs/base.py @@ -425,10 +425,10 @@ def copy( with self._lock: _src_path = self.validatepath(src_path) _dst_path = self.validatepath(dst_path) - if _src_path == _dst_path: - raise errors.IllegalDestination(dst_path) if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) + if _src_path == _dst_path: + raise errors.IllegalDestination(dst_path) with closing(self.open(_src_path, "rb")) as read_file: # FIXME(@althonos): typing complains because open return IO self.upload(_dst_path, read_file) # type: ignore diff --git a/fs/copy.py b/fs/copy.py index ae3acd8b..334c7ea9 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -257,9 +257,13 @@ def copy_file_internal( lock (bool): Lock both filesystems before copying. """ + _src_path = src_fs.validatepath(src_path) + _dst_path = dst_fs.validatepath(dst_path) if src_fs is dst_fs: - # Same filesystem, so we can do a potentially optimized - # copy + # It's not allowed to copy a file onto itself + if _src_path == _dst_path: + raise IllegalDestination(dst_path) + # Same filesystem, so we can do a potentially optimized copy src_fs.copy(src_path, dst_path, overwrite=True, preserve_time=preserve_time) return @@ -438,8 +442,6 @@ def copy_dir_if( dst_fs, create=True ) as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): - if src_fs == dst_fs and isbase(_src_path, _dst_path): - raise IllegalDestination(dst_path) _thread_safe = is_thread_safe(_src_fs, _dst_fs) with Copier( num_workers=workers if _thread_safe else 0, preserve_time=preserve_time diff --git a/fs/osfs.py b/fs/osfs.py index e56bde03..94d16844 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -408,12 +408,12 @@ def _check_copy(self, src_path, dst_path, overwrite=False): # check src_path exists and is a file if self.gettype(src_path) is not ResourceType.file: raise errors.FileExpected(src_path) - # it's not allowed to copy a file onto itself - if _src_path == _dst_path: - raise errors.IllegalDestination(dst_path) # check dst_path does not exist if we are not overwriting if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) + # it's not allowed to copy a file onto itself + if _src_path == _dst_path: + raise errors.IllegalDestination(dst_path) # check parent dir of _dst_path exists and is a directory if self.gettype(dirname(dst_path)) is not ResourceType.directory: raise errors.DirectoryExpected(dirname(dst_path)) diff --git a/fs/test.py b/fs/test.py index 154eb32e..32e6ea5c 100644 --- a/fs/test.py +++ b/fs/test.py @@ -1832,7 +1832,7 @@ def test_copy_file_onto_itself(self): self.fs.writetext("file.txt", "Hello") with self.assertRaises(errors.IllegalDestination): self.fs.copy("file.txt", "file.txt", overwrite=True) - with self.assertRaises(errors.IllegalDestination): + with self.assertRaises(errors.DestinationExists): self.fs.copy("file.txt", "file.txt", overwrite=False) self.assert_text("file.txt", "Hello") @@ -1841,7 +1841,7 @@ def test_copy_file_onto_itself_relpath(self): subdir.writetext("file.txt", "Hello") with self.assertRaises(errors.IllegalDestination): self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=True) - with self.assertRaises(errors.IllegalDestination): + with self.assertRaises(errors.DestinationExists): self.fs.copy("sub/file.txt", "sub/../sub/file.txt", overwrite=False) self.assert_text("sub/file.txt", "Hello") diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 2fbb7050..9e382031 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -270,8 +270,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): _val_src_path = self.validatepath(_src_path) _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): - if src_fs == dst_fs and _val_src_path == _val_dst_path: - raise errors.IllegalDestination(_dst_path) if not overwrite and dst_fs.exists(_dst_path): raise errors.DestinationExists(_dst_path) copy_file(src_fs, _src_path, dst_fs, _dst_path, preserve_time=preserve_time) From 6ef99f672a1c22d950ead87f837651f2002681b8 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 11:29:00 +0200 Subject: [PATCH 300/309] wrapfs does not need checks sprinkled everywhere anymore --- fs/copy.py | 11 ++++++++--- fs/wrapfs.py | 2 -- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 334c7ea9..e70aafa6 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -306,14 +306,19 @@ def copy_structure( dst_root (str): Path to the target root of the tree structure. """ + _src_root = abspath(normpath(src_root)) + _dst_root = abspath(normpath(dst_root)) + # It's not allowed to copy a structure into itself + if src_fs == dst_fs and isbase(_src_root, _dst_root): + raise IllegalDestination(dst_root) walker = walker or Walker() with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: with _src_fs.lock(), _dst_fs.lock(): - _dst_fs.makedirs(dst_root, recreate=True) - for dir_path in walker.dirs(_src_fs, src_root): + _dst_fs.makedirs(_dst_root, recreate=True) + for dir_path in walker.dirs(_src_fs, _src_root): _dst_fs.makedir( - combine(dst_root, frombase(src_root, dir_path)), recreate=True + combine(_dst_root, frombase(_src_root, dir_path)), recreate=True ) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 9e382031..e7f8f848 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -281,8 +281,6 @@ def copydir(self, src_path, dst_path, create=False, preserve_time=False): _val_src_path = self.validatepath(_src_path) _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): - if src_fs == dst_fs and isbase(_val_src_path, _val_dst_path): - raise errors.IllegalDestination(_dst_path) if not create and not dst_fs.exists(_dst_path): raise errors.ResourceNotFound(dst_path) if not src_fs.getinfo(_src_path).is_dir: From c703c99fe2ff1ac2ab98a8936d8535899eb1fc57 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 11:30:27 +0200 Subject: [PATCH 301/309] remove unneeded code --- fs/wrapfs.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index e7f8f848..2d166bc7 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -267,8 +267,6 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) - _val_src_path = self.validatepath(_src_path) - _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): if not overwrite and dst_fs.exists(_dst_path): raise errors.DestinationExists(_dst_path) @@ -278,8 +276,6 @@ def copydir(self, src_path, dst_path, create=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None src_fs, _src_path = self.delegate_path(src_path) dst_fs, _dst_path = self.delegate_path(dst_path) - _val_src_path = self.validatepath(_src_path) - _val_dst_path = self.validatepath(_dst_path) with unwrap_errors({_src_path: src_path, _dst_path: dst_path}): if not create and not dst_fs.exists(_dst_path): raise errors.ResourceNotFound(dst_path) From 300fe893a8a0cce445af6aee3fdf4a51805f16e1 Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 11:35:49 +0200 Subject: [PATCH 302/309] remove unused import --- fs/wrapfs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/wrapfs.py b/fs/wrapfs.py index 2d166bc7..abbbe4e3 100644 --- a/fs/wrapfs.py +++ b/fs/wrapfs.py @@ -12,7 +12,7 @@ from .copy import copy_dir, copy_file from .error_tools import unwrap_errors from .info import Info -from .path import abspath, isbase, join, normpath +from .path import abspath, join, normpath if typing.TYPE_CHECKING: from typing import ( From 9d9dcd675ba15821b37d827e07f56625ddf9e23d Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Wed, 27 Jul 2022 11:38:18 +0200 Subject: [PATCH 303/309] revert shutil formatting --- fs/osfs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/fs/osfs.py b/fs/osfs.py index 94d16844..7a095f7f 100644 --- a/fs/osfs.py +++ b/fs/osfs.py @@ -470,11 +470,7 @@ def copy(self, src_path, dst_path, overwrite=False, preserve_time=False): # type: (Text, Text, bool, bool) -> None with self._lock: _src_path, _dst_path = self._check_copy(src_path, dst_path, overwrite) - _src_sys, _dst_sys = ( - self.getsyspath(_src_path), - self.getsyspath(_dst_path), - ) - shutil.copy2(_src_sys, _dst_sys) + shutil.copy2(self.getsyspath(_src_path), self.getsyspath(_dst_path)) # --- Backport of os.scandir for Python < 3.5 ------------ From f169482233093332a7cb27e63c046d323838eb1f Mon Sep 17 00:00:00 2001 From: "M.Eng. Thomas Feldmann" Date: Wed, 27 Jul 2022 15:01:55 +0200 Subject: [PATCH 304/309] Set `timeout-minutes` in the `test.yml` Actions workflow (#548) --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f67ecfb..4be9098a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,7 @@ on: jobs: test: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: From 3bc8691ba07a031626c492de8ded014bd538ea1a Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Tue, 2 Aug 2022 11:35:20 +0200 Subject: [PATCH 305/309] use validatepath --- fs/base.py | 22 ++++++++++++---------- fs/copy.py | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/fs/base.py b/fs/base.py index 189d45ab..d42997d4 100644 --- a/fs/base.py +++ b/fs/base.py @@ -1171,17 +1171,19 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): ``dst_path`` does not exist. """ - if not overwrite and self.exists(dst_path): + _src_path = self.validatepath(src_path) + _dst_path = self.validatepath(dst_path) + if not overwrite and self.exists(_dst_path): raise errors.DestinationExists(dst_path) - if self.getinfo(src_path).is_dir: + if self.getinfo(_src_path).is_dir: raise errors.FileExpected(src_path) - if normpath(src_path) == normpath(dst_path): + if _src_path == _dst_path: # early exit when moving a file onto itself return if self.getmeta().get("supports_rename", False): try: - src_sys_path = self.getsyspath(src_path) - dst_sys_path = self.getsyspath(dst_path) + src_sys_path = self.getsyspath(_src_path) + dst_sys_path = self.getsyspath(_dst_path) except errors.NoSysPath: # pragma: no cover pass else: @@ -1191,15 +1193,15 @@ def move(self, src_path, dst_path, overwrite=False, preserve_time=False): pass else: if preserve_time: - copy_modified_time(self, src_path, self, dst_path) + copy_modified_time(self, _src_path, self, _dst_path) return with self._lock: - with self.open(src_path, "rb") as read_file: + with self.open(_src_path, "rb") as read_file: # FIXME(@althonos): typing complains because open return IO - self.upload(dst_path, read_file) # type: ignore + self.upload(_dst_path, read_file) # type: ignore if preserve_time: - copy_modified_time(self, src_path, self, dst_path) - self.remove(src_path) + copy_modified_time(self, _src_path, self, _dst_path) + self.remove(_src_path) def open( self, diff --git a/fs/copy.py b/fs/copy.py index e70aafa6..5f593571 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -306,8 +306,8 @@ def copy_structure( dst_root (str): Path to the target root of the tree structure. """ - _src_root = abspath(normpath(src_root)) - _dst_root = abspath(normpath(dst_root)) + _src_root = src_fs.validatepath(src_root) + _dst_root = dst_fs.validatepath(dst_root) # It's not allowed to copy a structure into itself if src_fs == dst_fs and isbase(_src_root, _dst_root): raise IllegalDestination(dst_root) From 420998dba31c25a1db642f161fb9a1efc4097a8f Mon Sep 17 00:00:00 2001 From: Thomas Feldmann Date: Tue, 2 Aug 2022 11:46:42 +0200 Subject: [PATCH 306/309] copy_structure FS can be of type str --- fs/copy.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/copy.py b/fs/copy.py index 5f593571..154fe715 100644 --- a/fs/copy.py +++ b/fs/copy.py @@ -306,14 +306,16 @@ def copy_structure( dst_root (str): Path to the target root of the tree structure. """ - _src_root = src_fs.validatepath(src_root) - _dst_root = dst_fs.validatepath(dst_root) - # It's not allowed to copy a structure into itself - if src_fs == dst_fs and isbase(_src_root, _dst_root): - raise IllegalDestination(dst_root) walker = walker or Walker() with manage_fs(src_fs) as _src_fs: with manage_fs(dst_fs, create=True) as _dst_fs: + _src_root = _src_fs.validatepath(src_root) + _dst_root = _dst_fs.validatepath(dst_root) + + # It's not allowed to copy a structure into itself + if _src_fs == _dst_fs and isbase(_src_root, _dst_root): + raise IllegalDestination(dst_root) + with _src_fs.lock(), _dst_fs.lock(): _dst_fs.makedirs(_dst_root, recreate=True) for dir_path in walker.dirs(_src_fs, _src_root): From 59f6e4d51a1983ca28d0b667bd1d6b3284ab5c56 Mon Sep 17 00:00:00 2001 From: "M.Eng. Thomas Feldmann" Date: Fri, 19 Aug 2022 11:17:37 +0200 Subject: [PATCH 307/309] Fix backward incompatibility introduced in 2.4.16 (#542) * add default overwrite arg (fixes #535) * update changelog * add tests * fix deletion when moving file on itself * compare normalized pathes in early exit * check no longer needed * remove unused import --- CHANGELOG.md | 3 +++ fs/move.py | 7 ++++++- tests/test_move.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ffe9392..ef16734e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Unreleased + ### Added - Added `filter_glob` and `exclude_glob` parameters to `fs.walk.Walker`. @@ -16,6 +17,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Fixed - Elaborated documentation of `filter_dirs` and `exclude_dirs` in `fs.walk.Walker`. Closes [#371](https://github.com/PyFilesystem/pyfilesystem2/issues/371). +- Fixes a backward incompatibility where `fs.move.move_file` raises `DestinationExists` + ([#535](https://github.com/PyFilesystem/pyfilesystem2/issues/535)). - Fixed a bug where files could be truncated or deleted when moved / copied onto itself. Closes [#546](https://github.com/PyFilesystem/pyfilesystem2/issues/546) diff --git a/fs/move.py b/fs/move.py index fdbe96fe..752b5816 100644 --- a/fs/move.py +++ b/fs/move.py @@ -82,7 +82,12 @@ def move_file( rel_dst = frombase(common, dst_syspath) with _src_fs.lock(), _dst_fs.lock(): with OSFS(common) as base: - base.move(rel_src, rel_dst, preserve_time=preserve_time) + base.move( + rel_src, + rel_dst, + overwrite=True, + preserve_time=preserve_time, + ) return # optimization worked, exit early except ValueError: # This is raised if we cannot find a common base folder. diff --git a/tests/test_move.py b/tests/test_move.py index 5401082e..8eb1af75 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -151,6 +151,45 @@ def test_move_file_read_only_mem_dest(self): dst_ro.exists("target.txt"), "file should not have been copied over" ) + @parameterized.expand([("temp", "temp://"), ("mem", "mem://")]) + def test_move_file_overwrite(self, _, fs_url): + # we use TempFS and MemoryFS in order to make sure the optimized code path + # behaves like the regular one (TempFS tests the optmized code path). + with open_fs(fs_url) as src, open_fs(fs_url) as dst: + src.writetext("file.txt", "source content") + dst.writetext("target.txt", "target content") + self.assertTrue(src.exists("file.txt")) + self.assertFalse(src.exists("target.txt")) + self.assertFalse(dst.exists("file.txt")) + self.assertTrue(dst.exists("target.txt")) + fs.move.move_file(src, "file.txt", dst, "target.txt") + self.assertFalse(src.exists("file.txt")) + self.assertFalse(src.exists("target.txt")) + self.assertFalse(dst.exists("file.txt")) + self.assertTrue(dst.exists("target.txt")) + self.assertEquals(dst.readtext("target.txt"), "source content") + + @parameterized.expand([("temp", "temp://"), ("mem", "mem://")]) + def test_move_file_overwrite_itself(self, _, fs_url): + # we use TempFS and MemoryFS in order to make sure the optimized code path + # behaves like the regular one (TempFS tests the optmized code path). + with open_fs(fs_url) as tmp: + tmp.writetext("file.txt", "content") + fs.move.move_file(tmp, "file.txt", tmp, "file.txt") + self.assertTrue(tmp.exists("file.txt")) + self.assertEquals(tmp.readtext("file.txt"), "content") + + @parameterized.expand([("temp", "temp://"), ("mem", "mem://")]) + def test_move_file_overwrite_itself_relpath(self, _, fs_url): + # we use TempFS and MemoryFS in order to make sure the optimized code path + # behaves like the regular one (TempFS tests the optmized code path). + with open_fs(fs_url) as tmp: + new_dir = tmp.makedir("dir") + new_dir.writetext("file.txt", "content") + fs.move.move_file(tmp, "dir/../dir/file.txt", tmp, "dir/file.txt") + self.assertTrue(tmp.exists("dir/file.txt")) + self.assertEquals(tmp.readtext("dir/file.txt"), "content") + @parameterized.expand([(True,), (False,)]) def test_move_file_cleanup_on_error(self, cleanup): with open_fs("mem://") as src, open_fs("mem://") as dst: From 8ed9dc495d8ba2f83fbb2a1145d34d92e13644be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jen=20H=C3=A4gg?= <66005238+jenhagg@users.noreply.github.com> Date: Tue, 18 Oct 2022 03:59:07 -0700 Subject: [PATCH 308/309] Update contributors (#554) --- CONTRIBUTORS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d23ba105..78102487 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -20,7 +20,7 @@ Many thanks to the following developers for contributing to this project: - [George Macon](https://github.com/gmacon) - [Giampaolo Cimino](https://github.com/gpcimino) - [@Hoboneer](https://github.com/Hoboneer) -- [Jon Hagg](https://github.com/jon-hagg) +- [Jen Hagg](https://github.com/jenhagg) - [Joseph Atkins-Turkish](https://github.com/Spacerat) - [Joshua Tauberer](https://github.com/JoshData) - [Justin Charlong](https://github.com/jcharlong) From 77a8562785fc37cb2e30bdcd39c133097ba62dce Mon Sep 17 00:00:00 2001 From: Andrew Scheller Date: Sat, 17 May 2025 14:12:52 +0100 Subject: [PATCH 309/309] Update docs link in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0c4fe2ba..0f1326b1 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ Python's Filesystem abstraction layer. ## Documentation -- [Wiki](https://www.pyfilesystem.org) -- [API Documentation](https://docs.pyfilesystem.org/) +- ~~[Wiki](https://www.pyfilesystem.org)~~ (currently offline) +- [API Documentation](https://pyfilesystem2.readthedocs.io/en/latest/) - [GitHub Repository](https://github.com/PyFilesystem/pyfilesystem2) - [Blog](https://www.willmcgugan.com/tag/fs/)