diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 10ba5faa..8d8aa551 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.12] + python-version: [3.13] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 562a7832..1ae19d7f 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -32,10 +32,10 @@ jobs: python-version: 3.12 - name: Install pypa/build and twine - run: python -m pip install --user build twine + run: python -m pip install --user --upgrade build twine pkginfo - name: Build a binary wheel and a source tarball - run: python -m build --sdist --wheel --outdir dist/ + run: python -m build --wheel --sdist --outdir dist/ - name: Validate wheel and sdis for Pypi run: python -m twine check dist/* @@ -72,6 +72,9 @@ jobs: needs: - create-gh-release runs-on: ubuntu-24.04 + environment: pypi-publish + permissions: + id-token: write steps: - name: Download built archives @@ -82,6 +85,4 @@ jobs: - name: Publish to PyPI if: startsWith(github.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.readthedocs.yml b/.readthedocs.yml index 683f3a82..27c15959 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.11" + python: "3.13" # Build PDF & ePub formats: diff --git a/README.rst b/README.rst index 1a4af20b..8a1a0b46 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ identify redistributable source code used in your project to help you comply with open source licenses conditions. This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.2 at: -https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/specification.html Build and tests status @@ -97,9 +97,12 @@ i.e. MAJOR.MINOR.PATCH format REFERENCE --------- -See https://aboutcode-toolkit.readthedocs.io/en/latest/ for documentation. +See https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/ +for documentation. -See https://aboutcode-toolkit.readthedocs.io/en/latest/reference.html for reference. +See +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/reference.html +for reference. TESTS and DEVELOPMENT --------------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7e3b8fec..475ea33b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,26 +13,58 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython - image_name: macOS-13 - python_versions: ["3.9", "3.10", "3.11"] - test_suites: - all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.13'] + test_suites: + all: make check - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ["3.9", "3.10", "3.11"] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml - parameters: - job_name: win2022_cpython - image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11"] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index 5ef0e063..6d317d4c 100755 --- a/configure +++ b/configure @@ -110,7 +110,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --wheel embed --pip embed --setuptools embed \ + --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ diff --git a/configure.bat b/configure.bat index 3e9881fb..15ab7015 100644 --- a/configure.bat +++ b/configure.bat @@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index a97f5a50..5863ccf5 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -1,205 +1,5 @@ -body { - color: #000000; - /* this is the font-family used in the SPATS wiki */ - /* font-family: -apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,Apple Color Emoji,Segoe UI Emoji; */ -} - -p { - margin-bottom: 10px; -} - -/* ul, ul.simple { - margin-bottom: 10px; -} */ - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 10px; -} - -h2, h3, h4, h5, h6 { - margin-top: 20px; - margin-top: 30px; -} - -h5 { - font-size: 17px; - font-size: 18px; - color: #666666; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 16px; - color: #666666; - color: #999999; - color: #778899; - color: #009999; - color: #006666; - color: #996633; - color: #009933; - color: #661aff; - /* color: #666699; */ - - /* font-style: italic; */ - /* font-weight: normal; */ - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #ffffff; - color: #000000; - background: #009900; - background: #00b33c; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #e8e8e8; - border: solid 1px #cccccc; - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #ffffff; - color: #000000; - background: #009900; - background: #00b33c; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #e8e8e8; - border: solid 1px #cccccc; - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff66; - background: #ffff99; - background: #fff3cd; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #e8e8e8; - border: solid 1px #cccccc; - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - /* color: #000000; */ - background: #3399ff; - background: #006bb3; - background: #cce5ff; - background: #b3d7ff; - background: #2196f3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #e8e8e8; - border: solid 1px #cccccc; - /* border: solid 1px #80bdff; */ - /* border: solid 1px #2196f3; */ - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - color: #cc0000; - background: #cce6ff; - background: #ffcc00; - background: #ffeb99; - background: #ccffff; - background: #ffd9b3; - background: #ffffff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; - border-bottom: solid 1px #ffcc00; - border-bottom: solid 1px #ffeb99; - border-bottom: solid 1px #e8e8e8; - border-bottom: solid 1px #ffd9b3; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #e8e8e8; - border: solid 1px #cccccc; - border: solid 1px #99ccff; - border: solid 1px #ffcc00; - border: solid 1px #ffeb99; - border: solid 1px #ffd9b3; - border: solid 1px #cc0000; - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - /* color: #ffffff; */ - background: #f5f5f5; - background: #e8e8e8; - background: #ffe6cc; - /* background: #606060; */ - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #f5f5f5; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - /* box-shadow: 5px 5px 18px #d8d8d8; */ - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - +/* this is the container for the pages */ .wy-nav-content { - /* max-width: 800px; */ - /* max-width: 1200px; */ max-width: 100%; padding: 0px 40px 0px 0px; margin-top: 0px; @@ -210,211 +10,17 @@ div.custom-admonition-examples.admonition { } div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - /* border: 1px solid #7fbbe3; */ - /* border: 1px solid #e7f2fa; */ - /* border: 1px solid #ffff99; */ - /* border: 1px solid #ccffcc; */ - /* border: 1px solid #f2f2f2; */ - /* border: 1px solid #e6f2ff; */ - /* border: 1px solid #fff3cd; */ - border: 1px solid #ccffff; - - /* background: #e7f2fa; */ - /* background: #e6ffff; */ - /* background: #ffff99; */ - /* background: #ccffcc; */ - /* background-color: #f2f2f2; */ - /* background: #e6f2ff; */ - /* background: #fff3cd; */ - background: #ccffff; - - /* font-size: 80%; */ - font-size: 100%; - /* font-size: 12px; */ - /* font-size: 14px; */ - /* font-size: 16px; */ - font-weight: 700; - font-weight: normal; - border-radius: 4px; - padding: 2.4px 6px; - padding: 4px 6px; - padding: 2px 0px; - margin: auto 2px; - - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} -/* .wy-table-responsive { - overflow: visible !important; -} */ - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px; - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - /* border: solid 1px #e8e8e8 !important; */ - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - /* border: solid 1px #e8e8e8 !important; */ - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 12px; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - /* line-height: 1.5; */ - overflow: visible; - white-space: pre-wrap; - /* color: #e74c3c; */ -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - border: 1px solid #e1e4e5; + max-width: 1300px; border: 0; - background-color: #f6f8fa; - background-color: #f8f8f8; - /* background-color: #f2f2f2; */ - border: solid 1px #e5e5e5; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #f8f8f8; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -/* span.pre { - color: #e74c3c; - color: #e01e5a; -} */ - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #6a6a6a; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { - margin-top: 0px; - margin-bottom: 0px; -} - -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 24px; - margin-bottom: 0px; - line-height: 24px; -} - -/* === */ - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #666666; - margin-top: 10px; - padding-top: 10px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; + padding: 10px 80px 10px 80px; + margin-left: 50px; } -div.genindex-jumpbox { - margin-bottom: 20px; +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } } diff --git a/docs/source/conf.py b/docs/source/conf.py index 348e6e98..b6aafba9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ # -- Project information ----------------------------------------------------- -project = 'aboutcode-toolkit' +project = "aboutcode-toolkit" copyright = "nexB Inc. and others." author = "AboutCode.org authors and contributors" diff --git a/docs/source/general.rst b/docs/source/general.rst index 36845ff5..35232b7b 100644 --- a/docs/source/general.rst +++ b/docs/source/general.rst @@ -44,7 +44,7 @@ Additional AboutCode Toolkit information is available at: - See :ref:`specification` for an overview and a link to the ABOUT File specification. -- https://github.com/nexB/aboutcode-toolkit/ for the AboutCode Toolkit tools. +- https://github.com/aboutcode-org/aboutcode-toolkit/ for the AboutCode Toolkit tools. Key Terminology =============== @@ -328,7 +328,7 @@ Prepare an Attribution Template to Use You can run attrib using the default_html.template (or default_json.template) provided with the AboutCode Toolkit tools: -https://github.com/nexB/aboutcode-toolkit/blob/develop/src/attributecode/templates/default_html.template +https://github.com/aboutcode-org/aboutcode-toolkit/blob/develop/src/attributecode/templates/default_html.template If you choose to do that, you will most likely want to edit the generated .html file to provide header information about your own organization and product. diff --git a/docs/source/home.rst b/docs/source/home.rst index 0e4f391e..414cc293 100644 --- a/docs/source/home.rst +++ b/docs/source/home.rst @@ -21,7 +21,7 @@ identify redistributable source code used in your project to help you comply with open source licenses conditions. This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.2 at: -https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/specification.html REQUIREMENTS @@ -50,7 +50,7 @@ Open and run the installer using all the default options. INSTALLATION ------------ Checkout or download and extract the AboutCode Toolkit from: - https://github.com/nexB/aboutcode-toolkit/ + https://github.com/aboutcode-org/aboutcode-toolkit/ To install all the needed dependencies in a virtualenv, run (on posix): ./configure @@ -84,9 +84,12 @@ i.e. MAJOR.MINOR.PATCH format REFERENCE --------- -See https://aboutcode-toolkit.readthedocs.io/en/latest/ for documentation. +See https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/ +for documentation. -See https://aboutcode-toolkit.readthedocs.io/en/latest/reference.html for reference. +See +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/reference.html +for reference. TESTS and DEVELOPMENT --------------------- @@ -111,23 +114,24 @@ HELP and SUPPORT ---------------- If you have a question or find a bug, enter a ticket at: - https://github.com/nexB/aboutcode-toolkit + https://github.com/aboutcode-org/aboutcode-toolkit For issues, you can use: - https://github.com/nexB/aboutcode-toolkit/issues + https://github.com/aboutcode-org/aboutcode-toolkit/issues SOURCE CODE ----------- The AboutCode Toolkit is available through GitHub. For the latest version visit: - https://github.com/nexB/aboutcode-toolkit + + https://github.com/aboutcode-org/aboutcode-toolkit HACKING ------- We accept pull requests provided under the same license as this tool. -You agree to the http://developercertificate.org/ +You agree to the https://developercertificate.org/ LICENSE diff --git a/docs/source/specification.rst b/docs/source/specification.rst index 0f273551..b6f500e5 100644 --- a/docs/source/specification.rst +++ b/docs/source/specification.rst @@ -47,8 +47,9 @@ The meaning of this ABOUT file is: - The file "httpd-2.4.3.tar.gz" is stored in the same directory and side-by-side with the ABOUT file "httpd-2.4.3.tar.gz.ABOUT" that documents it. - The name of this component is "Apache HTTP Server" with version "2.4.3". -- The homepage URL for this component is http://httpd.apache.org -- The file "httpd-2.4.3.tar.gz" was originally downloaded from http://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz +- The homepage URL for this component is https://httpd.apache.org +- The file "httpd-2.4.3.tar.gz" was originally downloaded from + https://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz - This component is licensed under "apache-2.0" - The licenses section contains the information of this "apache-2.0" license. - In the same directory, "apache-2.0.LICENSE" and "httpd.NOTICE" are files diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml index 85b611d3..d80e8dfb 100644 --- a/etc/ci/azure-container-deb.yml +++ b/etc/ci/azure-container-deb.yml @@ -21,7 +21,7 @@ jobs: - job: ${{ parameters.job_name }} pool: - vmImage: 'ubuntu-16.04' + vmImage: 'ubuntu-22.04' container: image: ${{ parameters.container }} diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml index 1e6657d0..a64138c9 100644 --- a/etc/ci/azure-container-rpm.yml +++ b/etc/ci/azure-container-rpm.yml @@ -1,6 +1,6 @@ parameters: job_name: '' - image_name: 'ubuntu-16.04' + image_name: 'ubuntu-22.04' container: '' python_path: '' python_version: '' diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index b9b2c0e7..f377578e 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -15,7 +15,7 @@ """ Utilities to manage requirements files and call pip. NOTE: this should use ONLY the standard library and not import anything else -because this is used for boostrapping with no requirements installed. +because this is used for bootstrapping with no requirements installed. """ @@ -31,7 +31,7 @@ def load_requirements(requirements_file="requirements.txt", with_unpinned=False) def get_required_name_versions(requirement_lines, with_unpinned=False): """ - Yield required (name, version) tuples given a`requirement_lines` iterable of + Yield required (name, version) tuples given a `requirement_lines` iterable of requirement text lines. Only accept requirements pinned to an exact version. """ @@ -47,7 +47,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): def get_required_name_version(requirement, with_unpinned=False): """ - Return a (name, version) tuple given a`requirement` specifier string. + Return a (name, version) tuple given a `requirement` specifier string. Requirement version must be pinned. If ``with_unpinned`` is True, unpinned requirements are accepted and only the name portion is returned. diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 84174e59..494fcb3b 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -24,13 +25,14 @@ import packageurl import requests import saneyaml -import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version +import utils_pip_compatibility_tags + """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. @@ -91,8 +93,7 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as - {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: @@ -114,12 +115,15 @@ TRACE_ULTRA_DEEP = False # Supported environments -PYTHON_VERSIONS = "39", "310", "311" +PYTHON_VERSIONS = "39", "310", "311", "312", "313", "314" PYTHON_DOT_VERSIONS_BY_VER = { "39": "3.9", "310": "3.10", "311": "3.11", + "312": "3.12", + "313": "3.13", + "314": "3.14", } @@ -131,10 +135,12 @@ def get_python_dot_version(version): ABIS_BY_PYTHON_VERSION = { - "37": ["cp37", "cp37m", "abi3"], - "38": ["cp38", "cp38m", "abi3"], "39": ["cp39", "cp39m", "abi3"], "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], + "312": ["cp312", "cp312m", "abi3"], + "313": ["cp313", "cp313m", "abi3"], + "314": ["cp314", "cp314m", "abi3"], } PLATFORMS_BY_OS = { @@ -552,8 +558,7 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - if not self.filename: - raise ValueError(f"self.filename has no value but is required: {self.filename!r}") + assert self.filename if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -821,9 +826,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): """ urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [eln for eln in extra_lic_names if eln] + extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -843,7 +848,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except Exception: + except: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -856,9 +861,8 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except Exception: - msg = f"No text for license {filename} in expression " - f"{self.license_expression!r} from {self}" + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' print(msg) errors.append(msg) @@ -998,7 +1002,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [url for url in urls if url.endswith(f"/{filename}")] + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1287,7 +1291,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except Exception: + except: return False @@ -1483,7 +1487,8 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - yield from self.wheels + for wheel in self.wheels: + yield wheel def get_url_for_filename(self, filename): """ @@ -1612,8 +1617,7 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " - "available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" ), ) @@ -1627,8 +1631,7 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. " - "Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." ), ) @@ -1637,8 +1640,7 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - if not name: - raise ValueError(f"name is required: {name!r}") + assert name normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1693,7 +1695,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [link.partition("#sha256=") for link in links] + links = [l.partition("#sha256=") for l in links] links = [url for url, _, _sha256 in links] return links @@ -1914,7 +1916,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2133,7 +2135,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: stdouts = [] @@ -2198,7 +2200,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print("Downloading wheels using command:", " ".join(cli_args)) + print(f"Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2231,7 +2233,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2282,5 +2284,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(lic).lower() for lic in declared_licenses] + lics = [python_safe_name(l).lower() for l in declared_licenses] return " AND ".join(lics).lower() diff --git a/pyproject.toml b/pyproject.toml index d79574ef..d3058aaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 6"] +requires = ["setuptools>=70.0.0", "wheel", "setuptools_scm[tomm]>=8.0"] build-backend = "setuptools.build_meta" [tool.setuptools_scm] @@ -68,7 +68,7 @@ include = [ "." ] -# ignore test data and testfiles: they should never be linted nor formatted +# ignore test data and testfiles: they should never be linted nor formatted exclude = [ # main style "**/tests/data/**/*", diff --git a/requirements-dev.txt b/requirements-dev.txt index 415e9d9e..87d1554f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,23 +1,49 @@ -bleach==4.1.0 -build==0.7.0 -commonmark==0.9.1 -docutils==0.18.1 -et-xmlfile==1.1.0 +alabaster==0.7.16 +anyio==4.6.0 +backports.tarfile==1.2.0 +babel==2.15.0 +doc8==1.1.2 +docutils==0.19 execnet==1.9.0 +h11==0.16.0 +id==1.5.0 +imagesize==1.4.1 iniconfig==1.1.1 +jaraco.classes==3.4.0 +jaraco.context==6.1.0 jeepney==0.7.1 keyring==23.4.1 -openpyxl==3.0.9 -pep517==0.12.0 -pkginfo==1.8.2 -py==1.11.0 -pytest==7.0.1 -pytest-forked==1.4.0 +markdown-it-py==3.0.0 +mdurl==0.1.2 +nh3==0.2.18 +pytest==9.0.2 pytest-xdist==2.5.0 readme-renderer==34.0 requests-toolbelt==0.9.1 +restructuredtext-lint==1.4.0 rfc3986==1.5.0 rich==12.3.0 +roman-numerals==3.1.0 +ruff==0.15.2 secretstorage==3.3.2 -tomli==1.2.3 +snowballstemmer==2.2.0 +Sphinx==7.2.6 +sphinx-autobuild==2024.2.4 +sphinx-copybutton==0.5.2 +sphinx-reredirects==0.1.3 +sphinx-rtd-dark-mode==1.3.0 +sphinx-rtd-theme==2.0.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-jquery==4.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +starlette==0.49.1 +stevedore==5.2.0 twine==3.8.0 +typing-extensions==4.12.0 +uvicorn==0.30.6 +watchfiles==0.24.0 +websockets==14.0 diff --git a/requirements.txt b/requirements.txt index 52362b57..c5220184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,80 +1,28 @@ -attrs==21.4.0 -banal==1.0.6 -beautifulsoup4==4.11.1 -binaryornot==0.4.4 -boolean.py==3.8 -certifi==2024.7.4 -cffi==1.15.0 -chardet==4.0.0 -charset-normalizer==2.0.12 -click==8.0.4 -colorama==0.4.4 -commoncode==30.2.0 -construct==2.10.68 -container-inspector==31.0.0 -cryptography==43.0.1 -debian-inspector==30.0.0 -dockerfile-parse==1.2.0 -dparse2==0.6.1 -extractcode==31.0.0 -extractcode-7z==16.5.210531 -extractcode-libarchive==3.5.1.210531 -fasteners==0.17.3 -fingerprints==1.0.3 -ftfy==6.0.3 -future==0.18.2 -gemfileparser==0.8.0 -html5lib==1.1 -idna==3.7 +attrs==25.4.0 +boolean.py==5.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +colorama==0.4.6 +commoncode==32.4.2 +cryptography==46.0.5 +et-xmlfile==2.0.0 +idna==3.11 importlib-metadata==4.8.3 -inflection==0.5.1 -intbitset==3.0.1 -isodate==0.6.1 -jaraco.functools==3.4.0 -javaproperties==0.8.1 -Jinja2==3.1.4 -jsonstreams==0.6.0 -license-expression==21.6.14 -lxml==4.9.1 -MarkupSafe==2.0.1 -more-itertools==8.13.0 -normality==2.3.3 -packagedcode-msitools==0.101.210706 -packageurl-python==0.9.9 -packaging==21.3 -parameter-expansion-patched==0.3.1 -patch==1.16 -pdfminer-six==20220506 -pefile==2021.9.3 -pip-requirements-parser==31.2.0 -pkginfo2==30.0.0 -pluggy==1.0.0 -plugincode==30.0.0 -ply==3.11 -publicsuffix2==2.20191221 -pyahocorasick==2.0.0b1 -pycparser==2.21 -pygmars==0.7.0 +jaraco.functools==4.4.0 +Jinja2==3.1.6 +license-expression==30.4.4 +MarkupSafe==3.0.3 +more-itertools==10.8.0 +openpyxl==3.1.5 +packageurl-python==0.17.6 +packaging==25.0 +pluggy==1.6.0 +pycparser==2.23 Pygments==2.15.0 -pymaven-patch==0.3.0 -pyparsing==3.0.8 -pytz==2022.1 -PyYAML==6.0 -rdflib==5.0.0 -regipy==2.3.1 -requests==2.31.0 -rpm-inspector-rpm==4.16.1.3.210404 -saneyaml==0.5.2 -six==1.16.0 -soupsieve==2.3.1 -spdx-tools==0.7.0rc0 -text-unidecode==1.3 -toml==0.10.2 -typecode==30.0.0 -typecode-libmagic==5.39.210531 -urllib3==1.26.19 -urlpy==0.5 -wcwidth==0.2.5 -webencodings==0.5.1 -xmltodict==0.12.0 -zipp==3.6.0 +PyYAML==6.0.3 +requests==2.32.5 +saneyaml==0.6.1 +urllib3==2.6.3 +zipp==3.23.0 diff --git a/setup.cfg b/setup.cfg index eb0d9f77..355101d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,8 +55,6 @@ packages = find: include_package_data = true zip_safe = false -setup_requires = setuptools_scm[toml] >= 4 - install_requires = attrs boolean.py >= 3.5 diff --git a/src/attributecode/__init__.py b/src/attributecode/__init__.py index 74e7215d..5450d44d 100644 --- a/src/attributecode/__init__.py +++ b/src/attributecode/__init__.py @@ -20,9 +20,9 @@ import saneyaml -__version__ = '11.1.1' +__version__ = "11.1.1" -__about_spec_version__ = '4.0.0' +__about_spec_version__ = "4.0.0" __copyright__ = """ Copyright (c) nexB Inc. All rights reserved. http://dejacode.org @@ -38,7 +38,7 @@ """ -class Error(namedtuple('Error', ['severity', 'message'])): +class Error(namedtuple("Error", ["severity", "message"])): """ An Error data with a severity and message. """ @@ -51,12 +51,11 @@ def __new__(self, severity, message): message = self._clean_string(repr(message)) message = message.strip('"') - return super(Error, self).__new__( - Error, severity, message) + return super(Error, self).__new__(Error, severity, message) def __repr__(self, *args, **kwargs): sev, msg = self._get_values() - return 'Error(%(sev)s, %(msg)s)' % locals() + return "Error(%(sev)s, %(msg)s)" % locals() def __eq__(self, other): return repr(self) == repr(other) @@ -68,7 +67,7 @@ def _get_values(self): def render(self): sev, msg = self._get_values() - return '%(sev)s: %(msg)s' % locals() + return "%(sev)s: %(msg)s" % locals() def to_dict(self, *args, **kwargs): """ @@ -85,7 +84,7 @@ def _clean_string(s): if not s: return s if s.startswith(('u"', "u'")): - s = s.lstrip('u') + s = s.lstrip("u") s = s.replace('[u"', '["') s = s.replace("[u'", "['") s = s.replace("(u'", "('") @@ -107,10 +106,10 @@ def _clean_string(s): NOTSET = 0 severities = { - CRITICAL: 'CRITICAL', - ERROR: 'ERROR', - WARNING: 'WARNING', - INFO: 'INFO', - DEBUG: 'DEBUG', - NOTSET: 'NOTSET' + CRITICAL: "CRITICAL", + ERROR: "ERROR", + WARNING: "WARNING", + INFO: "INFO", + DEBUG: "DEBUG", + NOTSET: "NOTSET", } diff --git a/src/attributecode/__main__.py b/src/attributecode/__main__.py index b1d85141..bae1bc5f 100644 --- a/src/attributecode/__main__.py +++ b/src/attributecode/__main__.py @@ -14,6 +14,7 @@ # limitations under the License. # ============================================================================ -if __name__ == '__main__': # pragma: nocover +if __name__ == "__main__": # pragma: nocover from attributecode import cmd + cmd.about() diff --git a/src/attributecode/api.py b/src/attributecode/api.py index 4763aa59..09579e42 100644 --- a/src/attributecode/api.py +++ b/src/attributecode/api.py @@ -36,18 +36,14 @@ def request_license_data(api_url, api_key, license_key): `license_key`. Send a request to `api_url` authenticating with `api_key`. """ headers = { - 'Authorization': 'Token %s' % api_key, - } - payload = { - 'api_key': api_key, - 'key': license_key, - 'format': 'json' + "Authorization": "Token %s" % api_key, } + payload = {"api_key": api_key, "key": license_key, "format": "json"} - api_url = api_url.rstrip('/') + api_url = api_url.rstrip("/") payload = urlencode(payload) - full_url = '%(api_url)s/?%(payload)s' % locals() + full_url = "%(api_url)s/?%(payload)s" % locals() # handle special characters in URL such as space etc. quoted_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]") @@ -58,23 +54,21 @@ def request_license_data(api_url, api_key, license_key): response_content = response.text # FIXME: this should be an ordered dict license_data = json.loads(response_content) - if not license_data.get('results', []): - msg = u"Invalid 'license': %s" % license_key + if not license_data.get("results", []): + msg = "Invalid 'license': %s" % license_key errors.append(Error(ERROR, msg)) except HTTPError as http_e: - msg = (u"Authorization denied. Invalid '--api_key'. " - u"License generation is skipped.") + msg = "Authorization denied. Invalid '--api_key'. License generation is skipped." errors.append(Error(ERROR, msg)) except Exception as e: # Already checked the authorization and accessible of the URL. # The only exception left is URL is accessible, but it's not a valid API URL - msg = (u"Invalid '--api_url'. " - u"License generation is skipped.") + msg = "Invalid '--api_url'. License generation is skipped." errors.append(Error(ERROR, msg)) finally: - if license_data.get('count') == 1: - license_data = license_data.get('results')[0] + if license_data.get("count") == 1: + license_data = license_data.get("results")[0] else: license_data = {} diff --git a/src/attributecode/attrib.py b/src/attributecode/attrib.py index b22c6d93..9f617b88 100644 --- a/src/attributecode/attrib.py +++ b/src/attributecode/attrib.py @@ -30,15 +30,19 @@ from attributecode.attrib_util import multi_sort DEFAULT_TEMPLATE_FILE = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'templates', 'default_html.template') + os.path.dirname(os.path.realpath(__file__)), "templates", "default_html.template" +) DEFAULT_TEMPLATE_SCANCODE_FILE = os.path.join( - os.path.dirname(os.path.realpath(__file__)), 'templates', 'scancode_html.template') + os.path.dirname(os.path.realpath(__file__)), "templates", "scancode_html.template" +) DEFAULT_LICENSE_SCORE = 100 -def generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=None, vartext=None): +def generate( + abouts, is_about_input, license_dict, scancode, min_license_score, template=None, vartext=None +): """ Generate an attribution text from an `abouts` list of About objects, a `template` template text and a `vartext` optional dict of extra @@ -53,9 +57,7 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, if template_error: lineno, message = template_error error = Error( - CRITICAL, - 'Template validation error at line: {lineno}: "{message}"'.format( - **locals()) + CRITICAL, 'Template validation error at line: {lineno}: "{message}"'.format(**locals()) ) errors.append(error) return error, None @@ -87,14 +89,13 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, filename = list(about.license_file.value.keys())[index] text = list(about.license_file.value.values())[index] else: - error = Error( - CRITICAL, 'No license file found for ' + name) + error = Error(CRITICAL, "No license file found for " + name) errors.append(error) break if about.license_url.value: url = about.license_url.value[index] else: - url = '' + url = "" license_object = License(key, name, filename, url, text) licenses_list.append(license_object) index = index + 1 @@ -114,7 +115,8 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, # The process will update the license_key, license_name and license_score. if scancode: abouts, meet_score_licenses_list = generate_sctk_input( - abouts, min_license_score, license_dict) + abouts, min_license_score, license_dict + ) # Remove the license object remove_list = [] for lic in licenses_list: @@ -126,7 +128,7 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, for about in abouts: # Create a license expression with license name - lic_name_expression = '' + lic_name_expression = "" lic_name_expression_list = [] if about.license_expression.value: for segment in about.license_expression.value.split(): @@ -139,12 +141,13 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, if not_lic: lic_name_expression_list.append(segment) # Join the license name expression into a single string - lic_name_expression = ' '.join(lic_name_expression_list) + lic_name_expression = " ".join(lic_name_expression_list) # Add the license name expression string into the about object as a custom field custom_field = StringField( - name='license_name_expression', value=lic_name_expression, present=True) - setattr(about, 'license_name_expression', custom_field) + name="license_name_expression", value=lic_name_expression, present=True + ) + setattr(about, "license_name_expression", custom_field) # Sort the about objects by name abouts = sorted(abouts, key=lambda x: x.name.value.lower()) @@ -158,7 +161,7 @@ def generate(abouts, is_about_input, license_dict, scancode, min_license_score, licenses_list=licenses_list, utcnow=utcnow, tkversion=__version__, - vartext=vartext + vartext=vartext, ) return errors, rendered @@ -193,9 +196,15 @@ def generate_sctk_input(abouts, min_license_score, license_dict): for key in lic_key: lic_name.append(license_dict[key][0]) lic_score = about.license_score.value - assert len(lic_key) == len(lic_name) - assert len(lic_key) == len(lic_score) - + len_lic_key = len(lic_key) + if len_lic_key != len(lic_name): + raise ValueError( + f"Mismatch between lengths: lic_key ({len_lic_key}) vs lic_name ({len(lic_name)})" + ) + if len_lic_key != len(lic_score): + raise ValueError( + f"Mismatch between lengths: lic_key ({len_lic_key}) vs lic_score ({len(lic_score)})" + ) lic_key_expression = about.license_key_expression.value if lic_key_expression: updated_lic_key_expression = [] @@ -205,13 +214,11 @@ def generate_sctk_input(abouts, min_license_score, license_dict): previous_score, _name = updated_dict[key] current_score = lic_score[index] if current_score > previous_score: - updated_dict[key] = ( - lic_score[index], lic_name[index]) + updated_dict[key] = (lic_score[index], lic_name[index]) # Track the duplicated index removed_index.append(index) else: - updated_dict[key] = ( - lic_score[index], lic_name[index]) + updated_dict[key] = (lic_score[index], lic_name[index]) updated_lic_key_expression.append(key) # Remove the duplication for index, key in enumerate(about.license_key.value): @@ -225,8 +232,7 @@ def generate_sctk_input(abouts, min_license_score, license_dict): updated_lic_name = [] updated_lic_score = [] for index, lic in enumerate(updated_dict): - _sp_char, lic_keys, _invalid_lic_exp = parse_license_expression( - lic) + _sp_char, lic_keys, _invalid_lic_exp = parse_license_expression(lic) score, name = updated_dict[lic] if score >= min_license_score: for lic_key in lic_keys: @@ -253,10 +259,10 @@ def generate_sctk_input(abouts, min_license_score, license_dict): def get_license_file_key(license_text_name): - if license_text_name.endswith('.LICENSE'): + if license_text_name.endswith(".LICENSE"): # See https://github.com/aboutcode-org/aboutcode-toolkit/issues/439 # for why using split instead of strip - return license_text_name.rsplit('.', 1)[0] + return license_text_name.rsplit(".", 1)[0] else: return license_text_name @@ -267,13 +273,21 @@ def check_template(template_string): message) if the template is invalid or None if it is valid. """ try: - jinja2.filters.FILTERS['multi_sort'] = multi_sort + jinja2.filters.FILTERS["multi_sort"] = multi_sort jinja2.Template(template_string) except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e: return e.lineno, e.message -def generate_from_file(abouts, is_about_input, license_dict, scancode, min_license_score, template_loc=None, vartext=None): +def generate_from_file( + abouts, + is_about_input, + license_dict, + scancode, + min_license_score, + template_loc=None, + vartext=None, +): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `vartext` optional @@ -289,12 +303,29 @@ def generate_from_file(abouts, is_about_input, license_dict, scancode, min_licen template_loc = add_unc(DEFAULT_TEMPLATE_FILE) else: template_loc = add_unc(template_loc) - with open(template_loc, encoding='utf-8', errors='replace') as tplf: + with open(template_loc, encoding="utf-8", errors="replace") as tplf: tpls = tplf.read() - return generate(abouts, is_about_input, license_dict, scancode, min_license_score, template=tpls, vartext=vartext) + return generate( + abouts, + is_about_input, + license_dict, + scancode, + min_license_score, + template=tpls, + vartext=vartext, + ) -def generate_and_save(abouts, is_about_input, license_dict, output_location, scancode=False, min_license_score=0, template_loc=None, vartext=None): +def generate_and_save( + abouts, + is_about_input, + license_dict, + output_location, + scancode=False, + min_license_score=0, + template_loc=None, + vartext=None, +): """ Generate an attribution text from an `abouts` list of About objects, a `template_loc` template file location and a `vartext` optional @@ -308,14 +339,15 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, sca if not about.license_expression.value: continue special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - about.license_expression.value) + about.license_expression.value + ) if special_char_in_expression or invalid_lic_exp: if special_char_in_expression: - msg = (u"The following character(s) cannot be in the license_expression: " + - str(special_char_in_expression)) + msg = "The following character(s) cannot be in the license_expression: " + str( + special_char_in_expression + ) else: - msg = (u"This license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) rendering_error, rendered = generate_from_file( @@ -329,11 +361,11 @@ def generate_and_save(abouts, is_about_input, license_dict, output_location, sca ) if rendering_error: - errors.append(rendering_error) + errors.extend(rendering_error) if rendered: output_location = add_unc(output_location) - with open(output_location, 'w', encoding='utf-8', errors='replace') as of: + with open(output_location, "w", encoding="utf-8", errors="replace") as of: of.write(rendered) return errors, rendered diff --git a/src/attributecode/attrib_util.py b/src/attributecode/attrib_util.py index 3b919abf..9431076a 100644 --- a/src/attributecode/attrib_util.py +++ b/src/attributecode/attrib_util.py @@ -15,6 +15,7 @@ # ============================================================================ from jinja2 import Environment + try: from jinja2.filters import pass_environment except ImportError: @@ -35,15 +36,12 @@ def get_template(template_text): """ env = Environment(autoescape=True) # register our custom filters - env.filters.update(dict( - unique_together=unique_together, - multi_sort=multi_sort)) + env.filters.update(dict(unique_together=unique_together, multi_sort=multi_sort)) return env.from_string(template_text) @pass_environment -def multi_sort(environment, value, reverse=False, case_sensitive=False, - attributes=None): +def multi_sort(environment, value, reverse=False, case_sensitive=False, attributes=None): """ Sort an iterable using an "attributes" list of attribute names available on each iterable item. Sort ascending unless reverse is "true". Ignore the case @@ -57,9 +55,10 @@ def multi_sort(environment, value, reverse=False, case_sensitive=False, """ if not attributes: raise FilterArgumentError( - 'The multi_sort filter requires a list of attributes as argument, ' - 'such as in: ' - "for item in iterable|multi_sort(attributes=['date', 'name'])") + "The multi_sort filter requires a list of attributes as argument, " + "such as in: " + "for item in iterable|multi_sort(attributes=['date', 'name'])" + ) # build a list of attribute getters, one for each attribute do_ignore_case = ignore_case if not case_sensitive else None @@ -91,9 +90,10 @@ def unique_together(environment, value, case_sensitive=False, attributes=None): """ if not attributes: raise FilterArgumentError( - 'The unique_together filter requires a list of attributes as argument, ' - 'such as in: ' - "{% for item in iterable|unique_together(attributes=['date', 'name']) %} ") + "The unique_together filter requires a list of attributes as argument, " + "such as in: " + "{% for item in iterable|unique_together(attributes=['date', 'name']) %} " + ) # build a list of attribute getters, one for each attribute do_ignore_case = ignore_case if not case_sensitive else None diff --git a/src/attributecode/cmd.py b/src/attributecode/cmd.py index 5f2f5047..3f61b89c 100644 --- a/src/attributecode/cmd.py +++ b/src/attributecode/cmd.py @@ -30,7 +30,11 @@ from attributecode.model import pre_process_and_fetch_license_dict from attributecode.model import get_copy_list from attributecode.model import copy_redist_src -from attributecode.model import collect_inventory, collect_abouts_license_expression, collect_inventory_license_expression +from attributecode.model import ( + collect_inventory, + collect_abouts_license_expression, + collect_inventory_license_expression, +) from attributecode.gen import generate as generate_about_files, load_inventory from attributecode.attrib import generate_and_save as generate_attribution_doc from attributecode.attrib import DEFAULT_LICENSE_SCORE @@ -65,17 +69,17 @@ See the License for the specific language governing permissions and limitations under the License.""" -prog_name = 'AboutCode-toolkit' +prog_name = "AboutCode-toolkit" -intro = '''%(prog_name)s version %(__version__)s +intro = """%(prog_name)s version %(__version__)s ABOUT spec version: %(__about_spec_version__)s https://aboutcode.org %(__copyright__)s -''' % locals() +""" % locals() def print_version(): - click.echo('Running aboutcode-toolkit version ' + __version__) + click.echo("Running aboutcode-toolkit version " + __version__) class AboutCommand(click.Command): @@ -83,29 +87,34 @@ class AboutCommand(click.Command): An enhanced click Command working around some Click quirk. """ - def main(self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra): + def main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra): """ Workaround click bug https://github.com/mitsuhiko/click/issues/365 """ return click.Command.main( - self, args=args, prog_name=self.name, - complete_var=complete_var, standalone_mode=standalone_mode, **extra) + self, + args=args, + prog_name=self.name, + complete_var=complete_var, + standalone_mode=standalone_mode, + **extra, + ) # we define a main entry command with subcommands -@click.group(name='about') +@click.group(name="about") @click.version_option(version=__version__, prog_name=prog_name, message=intro) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def about(): """ -Generate licensing attribution and credit notices from .ABOUT files and inventories. + Generate licensing attribution and credit notices from .ABOUT files and inventories. -Read, write and collect provenance and license inventories from .ABOUT files to and from JSON or CSV files. + Read, write and collect provenance and license inventories from .ABOUT files to and from JSON or CSV files. -Use about --help for help on a command. + Use about --help for help on a command. """ + ###################################################################### # option validators ###################################################################### @@ -121,168 +130,201 @@ def validate_key_values(ctx, param, value): kvals, errors = parse_key_values(value) if errors: - ive = '\n'.join(sorted(' ' + x for x in errors)) - msg = ('Invalid {param} option(s):\n' - '{ive}'.format(**locals())) + ive = "\n".join(sorted(" " + x for x in errors)) + msg = "Invalid {param} option(s):\n{ive}".format(**locals()) raise click.UsageError(msg) return kvals -def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))): +def validate_extensions( + ctx, + param, + value, + extensions=tuple( + ( + ".csv", + ".json", + ) + ), +): if not value: return if not value.endswith(extensions): - msg = ' '.join(extensions) + msg = " ".join(extensions) raise click.UsageError( - 'Invalid {param} file extension: must be one of: {msg}'.format(**locals())) + "Invalid {param} file extension: must be one of: {msg}".format(**locals()) + ) return value + ###################################################################### # inventory subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file or stdout.') -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - metavar='OUTPUT') -@click.option('--exclude', - multiple=True, - metavar='PATTERN', - help='Exclude the processing of the specified input pattern (e.g. *tests* or test/).') -@click.option('-f', '--format', - is_flag=False, - default='csv', - show_default=True, - type=click.Choice(['json', 'csv', 'excel']), - help='Set OUTPUT inventory file format.') -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') +@about.command( + cls=AboutCommand, + short_help="Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file or stdout.", +) +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument("output", required=True, metavar="OUTPUT") +@click.option( + "--exclude", + multiple=True, + metavar="PATTERN", + help="Exclude the processing of the specified input pattern (e.g. *tests* or test/).", +) +@click.option( + "-f", + "--format", + is_flag=False, + default="csv", + show_default=True, + type=click.Choice(["json", "csv", "excel"]), + help="Set OUTPUT inventory file format.", +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") def inventory(location, output, exclude, format, quiet, verbose): # NOQA """ -Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file. + Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file. -LOCATION: Path to an ABOUT file or a directory with ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. -OUTPUT: Path to the CSV/JSON/XLSX inventory file to create, or -using '-' to print result on screen/to stdout (Excel-formatted output -cannot be used in stdout). + OUTPUT: Path to the CSV/JSON/XLSX inventory file to create, or + using '-' to print result on screen/to stdout (Excel-formatted output + cannot be used in stdout). """ # We are not using type=click.Path() to validate the output location as # it does not support `-` , which is used to print the result to stdout. - if not output == '-': + if not output == "-": parent_dir = os.path.dirname(output) if not os.path.exists(parent_dir): - msg = 'The OUTPUT directory: {parent_dir} does not exist.'.format(**locals()) - msg += '\nPlease correct and re-run' + msg = "The OUTPUT directory: {parent_dir} does not exist.".format(**locals()) + msg += "\nPlease correct and re-run" click.echo(msg) sys.exit(1) else: # Check the format if output is stdout as xlsx format cannot be displayed. - if format == 'excel': - msg = 'Excel-formatted output cannot be used in stdout.' + if format == "excel": + msg = "Excel-formatted output cannot be used in stdout." click.echo(msg) sys.exit(0) if not quiet: print_version() - click.echo('Collecting inventory from ABOUT files...') + click.echo("Collecting inventory from ABOUT files...") - if location.lower().endswith('.zip'): + if location.lower().endswith(".zip"): # accept zipped ABOUT files as input location = extract_zip(location) errors, abouts = collect_inventory(location, exclude) write_output(abouts=abouts, location=output, format=format) - if output == '-': + if output == "-": log_file_loc = None else: - log_file_loc = output + '-error.log' - errors_count = report_errors( - errors, quiet, verbose, log_file_loc) - if not quiet and not output == '-': - msg = 'Inventory collected in {output}.'.format(**locals()) + log_file_loc = output + "-error.log" + errors_count = report_errors(errors, quiet, verbose, log_file_loc) + if not quiet and not output == "-": + msg = "Inventory collected in {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) + ###################################################################### # gen subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Generate .ABOUT files from an inventory as CSV/JSON/XLSX.') -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - metavar='OUTPUT', - type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True)) -@click.option('--android', - is_flag=True, - help='Generate MODULE_LICENSE_XXX (XXX will be replaced by license key) and NOTICE ' - 'as the same design as from Android.') +@about.command( + cls=AboutCommand, short_help="Generate .ABOUT files from an inventory as CSV/JSON/XLSX." +) +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", + required=True, + metavar="OUTPUT", + type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--android", + is_flag=True, + help="Generate MODULE_LICENSE_XXX (XXX will be replaced by license key) and NOTICE " + "as the same design as from Android.", +) # FIXME: the CLI UX should be improved with two separate options for API key and URL -@click.option('--fetch-license', - is_flag=True, - help='Fetch license data and text files from the ScanCode LicenseDB.') -@click.option('--fetch-license-djc', - nargs=2, - type=str, - metavar='api_url api_key', - help='Fetch license data and text files from a DejaCode License Library ' - 'API URL using the API KEY.') -@click.option('--scancode', - is_flag=True, - help='Indicate the input JSON file is from scancode_toolkit.') -@click.option('--reference', - metavar='DIR', - type=click.Path(exists=True, file_okay=False, - readable=True, resolve_path=True), - help='Path to a directory with reference license data and text files.') -@click.option('--worksheet', - metavar='name', - help='The worksheet name from the INPUT. (Default: the "active" worksheet)') -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') -def gen(location, output, android, fetch_license, fetch_license_djc, scancode, reference, worksheet, quiet, verbose): +@click.option( + "--fetch-license", + is_flag=True, + help="Fetch license data and text files from the ScanCode LicenseDB.", +) +@click.option( + "--fetch-license-djc", + nargs=2, + type=str, + metavar="api_url api_key", + help="Fetch license data and text files from a DejaCode License Library " + "API URL using the API KEY.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--reference", + metavar="DIR", + type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True), + help="Path to a directory with reference license data and text files.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def gen( + location, + output, + android, + fetch_license, + fetch_license_djc, + scancode, + reference, + worksheet, + quiet, + verbose, +): """ -Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location. + Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location. -LOCATION: Path to a JSON/CSV/XLSX inventory file. + LOCATION: Path to a JSON/CSV/XLSX inventory file. -OUTPUT: Path to a directory where ABOUT files are generated. + OUTPUT: Path to a directory where ABOUT files are generated. """ if not quiet: print_version() - click.echo('Generating .ABOUT files...') + click.echo("Generating .ABOUT files...") # FIXME: This should be checked in the `click` - if not location.endswith(('.csv', '.json', '.xlsx')): + if not location.endswith((".csv", ".json", ".xlsx")): raise click.UsageError( - 'ERROR: Invalid input file extension: must be one .csv or .json or .xlsx.') + "ERROR: Invalid input file extension: must be one .csv or .json or .xlsx." + ) - if worksheet and not location.endswith('.xlsx'): - raise click.UsageError( - 'ERROR: --worksheet option only works with .xlsx input.') + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") errors, abouts = generate_about_files( location=location, @@ -292,15 +334,13 @@ def gen(location, output, android, fetch_license, fetch_license_djc, scancode, r fetch_license=fetch_license, fetch_license_djc=fetch_license_djc, scancode=scancode, - worksheet=worksheet + worksheet=worksheet, ) - errors_count = report_errors( - errors, quiet, verbose, log_file_loc=output + '-error.log') + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet: abouts_count = len(abouts) - msg = '{abouts_count} .ABOUT files generated in {output}.'.format( - **locals()) + msg = "{abouts_count} .ABOUT files generated in {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) @@ -309,57 +349,66 @@ def gen(location, output, android, fetch_license, fetch_license_djc, scancode, r # gen_license subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Fetch and save all the licenses in the license_expression field to a directory.') -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - metavar='OUTPUT', - type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True)) -@click.option('--djc', - nargs=2, - type=str, - metavar='api_url api_key', - help='Fetch licenses from a DejaCode License Library.') -@click.option('--scancode', - is_flag=True, - help='Indicate the input JSON file is from scancode_toolkit.') -@click.option('--worksheet', - metavar='name', - help='The worksheet name from the INPUT. (Default: the "active" worksheet)') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') + +@about.command( + cls=AboutCommand, + short_help="Fetch and save all the licenses in the license_expression field to a directory.", +) +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", + required=True, + metavar="OUTPUT", + type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--djc", + nargs=2, + type=str, + metavar="api_url api_key", + help="Fetch licenses from a DejaCode License Library.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") def gen_license(location, output, djc, scancode, worksheet, verbose): """ -Fetch licenses (Default: ScanCode LicenseDB) in the license_expression field and save to the output location. + Fetch licenses (Default: ScanCode LicenseDB) in the license_expression field and save to the output location. -LOCATION: Path to a JSON/CSV/XLSX/.ABOUT file(s) + LOCATION: Path to a JSON/CSV/XLSX/.ABOUT file(s) -OUTPUT: Path to a directory where license files are saved. + OUTPUT: Path to a directory where license files are saved. """ print_version() - api_url = '' - api_key = '' + api_url = "" + api_key = "" errors = [] - if worksheet and not location.endswith('.xlsx'): - raise click.UsageError( - 'ERROR: --worksheet option only works with .xlsx input.') + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") - log_file_loc = os.path.join(output, 'error.log') + log_file_loc = os.path.join(output, "error.log") - if location.endswith('.csv') or location.endswith('.json') or location.endswith('.xlsx'): + if location.endswith(".csv") or location.endswith(".json") or location.endswith(".xlsx"): errors, abouts = collect_inventory_license_expression( - location=location, scancode=scancode, worksheet=worksheet) + location=location, scancode=scancode, worksheet=worksheet + ) if errors: severe_errors_count = report_errors( - errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc) + errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc + ) sys.exit(severe_errors_count) else: # _errors, abouts = collect_inventory(location) @@ -370,10 +419,11 @@ def gen_license(location, output, djc, scancode, worksheet, verbose): api_url = djc[0].strip("'").strip('"') api_key = djc[1].strip("'").strip('"') - click.echo('Fetching licenses...') + click.echo("Fetching licenses...") from_check = False license_dict, lic_errors = pre_process_and_fetch_license_dict( - abouts, from_check, api_url, api_key, scancode) + abouts, from_check, api_url, api_key, scancode + ) if lic_errors: errors.extend(lic_errors) @@ -391,7 +441,8 @@ def gen_license(location, output, djc, scancode, worksheet, verbose): errors.extend(write_errors) severe_errors_count = report_errors( - errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc) + errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc + ) sys.exit(severe_errors_count) @@ -404,103 +455,123 @@ def validate_template(ctx, param, value): if not value: return None - with open(value, encoding='utf-8', errors='replace') as templatef: + with open(value, encoding="utf-8", errors="replace") as templatef: template_error = check_template(templatef.read()) if template_error: lineno, message = template_error raise click.UsageError( - 'Template syntax error at line: ' - '{lineno}: "{message}"'.format(**locals())) + 'Template syntax error at line: {lineno}: "{message}"'.format(**locals()) + ) return value -@about.command(cls=AboutCommand, - short_help='Generate an attribution document from JSON/CSV/XLSX/.ABOUT files.') -@click.argument('input', - required=True, - metavar='INPUT', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - metavar='OUTPUT', - type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) -@click.option('--api_url', - nargs=1, - type=click.STRING, - metavar='URL', - help='URL to DejaCode License Library.') -@click.option('--api_key', - nargs=1, - type=click.STRING, - metavar='KEY', - help='API Key for the DejaCode License Library') -@click.option('--min-license-score', - type=int, - help='Attribute components that have license score higher than or equal to the defined ' - '--min-license-score.') -@click.option('--scancode', - is_flag=True, - help='Indicate the input JSON file is from scancode_toolkit.') -@click.option('--reference', - metavar='DIR', - type=click.Path(exists=True, file_okay=False, - readable=True, resolve_path=True), - help='Path to a directory with reference files where "license_file" and/or "notice_file"' - ' located.') -@click.option('--template', - metavar='FILE', - callback=validate_template, - type=click.Path(exists=True, dir_okay=False, - readable=True, resolve_path=True), - help='Path to an optional custom attribution template to generate the ' - 'attribution document. If not provided the default built-in template is used.') -@click.option('--vartext', - multiple=True, - callback=validate_key_values, - metavar='=', - help='Add variable text as key=value for use in a custom attribution template.') -@click.option('--worksheet', - metavar='name', - help='The worksheet name from the INPUT. (Default: the "active" worksheet)') -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') -def attrib(input, output, api_url, api_key, scancode, min_license_score, reference, template, vartext, worksheet, quiet, verbose): +@about.command( + cls=AboutCommand, short_help="Generate an attribution document from JSON/CSV/XLSX/.ABOUT files." +) +@click.argument( + "input", + required=True, + metavar="INPUT", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", + required=True, + metavar="OUTPUT", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--api_url", nargs=1, type=click.STRING, metavar="URL", help="URL to DejaCode License Library." +) +@click.option( + "--api_key", + nargs=1, + type=click.STRING, + metavar="KEY", + help="API Key for the DejaCode License Library", +) +@click.option( + "--min-license-score", + type=int, + help="Attribute components that have license score higher than or equal to the defined " + "--min-license-score.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--reference", + metavar="DIR", + type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True), + help='Path to a directory with reference files where "license_file" and/or "notice_file"' + " located.", +) +@click.option( + "--template", + metavar="FILE", + callback=validate_template, + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), + help="Path to an optional custom attribution template to generate the " + "attribution document. If not provided the default built-in template is used.", +) +@click.option( + "--vartext", + multiple=True, + callback=validate_key_values, + metavar="=", + help="Add variable text as key=value for use in a custom attribution template.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def attrib( + input, + output, + api_url, + api_key, + scancode, + min_license_score, + reference, + template, + vartext, + worksheet, + quiet, + verbose, +): """ -Generate an attribution document at OUTPUT using JSON, CSV or XLSX or .ABOUT files at INPUT. + Generate an attribution document at OUTPUT using JSON, CSV or XLSX or .ABOUT files at INPUT. -INPUT: Path to a file (.ABOUT/.csv/.json/.xlsx), directory or .zip archive containing .ABOUT files. + INPUT: Path to a file (.ABOUT/.csv/.json/.xlsx), directory or .zip archive containing .ABOUT files. -OUTPUT: Path where to write the attribution document. + OUTPUT: Path where to write the attribution document. """ # A variable to define if the input ABOUT file(s) is_about_input = False - rendered = '' + rendered = "" license_dict = {} errors = [] - if worksheet and not input.endswith('.xlsx'): - raise click.UsageError( - 'ERROR: --worksheet option only works with .xlsx input.') + if worksheet and not input.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") if not quiet: print_version() - click.echo('Generating attribution...') + click.echo("Generating attribution...") # accept zipped ABOUT files as input - if input.lower().endswith('.zip'): + if input.lower().endswith(".zip"): input = extract_zip(input) if scancode: - if not input.endswith('.json'): - msg = 'The input file from scancode toolkit needs to be in JSON format.' + if not input.endswith(".json"): + msg = "The input file from scancode toolkit needs to be in JSON format." click.echo(msg) sys.exit(1) if not min_license_score and not min_license_score == 0: @@ -508,12 +579,14 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen if min_license_score: if not scancode: - msg = ('This option requires a JSON file generated by scancode toolkit as the input. ' + - 'The "--scancode" option is required.') + msg = ( + "This option requires a JSON file generated by scancode toolkit as the input. " + + 'The "--scancode" option is required.' + ) click.echo(msg) sys.exit(1) - if input.endswith('.json') or input.endswith('.csv') or input.endswith('.xlsx'): + if input.endswith(".json") or input.endswith(".csv") or input.endswith(".xlsx"): is_about_input = False from_attrib = True if not reference: @@ -528,13 +601,13 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen from_attrib=from_attrib, scancode=scancode, reference_dir=reference, - worksheet=worksheet + worksheet=worksheet, ) # Exit if CRITICAL error if errors: for e in errors: - if severities[e.severity] == 'CRITICAL': + if severities[e.severity] == "CRITICAL": click.echo(e) sys.exit(1) @@ -543,7 +616,7 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen _errors, abouts = collect_inventory(input) if not abouts: - msg = 'No ABOUT file or reference is found from the input. Attribution generation halted.' + msg = "No ABOUT file or reference is found from the input. Attribution generation halted." click.echo(msg) errors_count = 1 sys.exit(errors_count) @@ -560,13 +633,14 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen click.echo(msg) sys.exit(1) else: - api_url = '' - api_key = '' + api_url = "" + api_key = "" api_url = api_url.strip("'").strip('"') api_key = api_key.strip("'").strip('"') from_check = False license_dict, lic_errors = pre_process_and_fetch_license_dict( - abouts, from_check, api_url, api_key, scancode, reference) + abouts, from_check, api_url, api_key, scancode, reference + ) errors.extend(lic_errors) sorted_license_dict = sorted(license_dict) @@ -574,8 +648,7 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen for about in abouts: if about.license_file.value or about.notice_file.value: if not reference: - msg = ( - '"license_file" / "notice_file" field contains value. Use `--reference` to indicate its parent directory.') + msg = '"license_file" / "notice_file" field contains value. Use `--reference` to indicate its parent directory.' click.echo(msg) # sys.exit(1) @@ -592,72 +665,63 @@ def attrib(input, output, api_url, api_key, scancode, min_license_score, referen ) errors.extend(attrib_errors) - errors_count = report_errors( - errors, quiet, verbose, log_file_loc=output + '-error.log') + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet: if rendered: - msg = 'Attribution generated in: {output}'.format(**locals()) + msg = "Attribution generated in: {output}".format(**locals()) click.echo(msg) else: - msg = 'Attribution generation failed.' + msg = "Attribution generation failed." click.echo(msg) sys.exit(errors_count) + ###################################################################### # collect_redist_src subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Collect redistributable sources.') -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - metavar='OUTPUT') -@click.option('--from-inventory', - metavar='FILE', - type=click.Path(exists=True, dir_okay=False, - readable=True, resolve_path=True), - help='Path to an inventory CSV/JSON/XLSX file as the base list for files/directories ' - 'that need to be copied which have the \'redistribute\' flagged.') -@click.option('--with-structures', - is_flag=True, - help='Copy sources with directory structure.') -@click.option('--zip', - is_flag=True, - help='Zip the copied sources to the output location.') -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') +@about.command(cls=AboutCommand, short_help="Collect redistributable sources.") +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument("output", required=True, metavar="OUTPUT") +@click.option( + "--from-inventory", + metavar="FILE", + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), + help="Path to an inventory CSV/JSON/XLSX file as the base list for files/directories " + "that need to be copied which have the 'redistribute' flagged.", +) +@click.option("--with-structures", is_flag=True, help="Copy sources with directory structure.") +@click.option("--zip", is_flag=True, help="Zip the copied sources to the output location.") +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") def collect_redist_src(location, output, from_inventory, with_structures, zip, quiet, verbose): """ -Collect sources that have 'redistribute' flagged as 'True' in .ABOUT files or inventory -to the output location. + Collect sources that have 'redistribute' flagged as 'True' in .ABOUT files or inventory + to the output location. -LOCATION: Path to a directory containing sources that need to be copied -(and containing ABOUT files if `inventory` is not provided) + LOCATION: Path to a directory containing sources that need to be copied + (and containing ABOUT files if `inventory` is not provided) -OUTPUT: Path to a directory or a zip file where sources will be copied to. + OUTPUT: Path to a directory or a zip file where sources will be copied to. """ if zip: - if not output.endswith('.zip'): - click.echo('The output needs to be a zip file.') + if not output.endswith(".zip"): + click.echo("The output needs to be a zip file.") sys.exit() if not quiet: print_version() - click.echo('Collecting inventory from ABOUT files...') + click.echo("Collecting inventory from ABOUT files...") - if location.lower().endswith('.zip'): + if location.lower().endswith(".zip"): # accept zipped ABOUT files as input location = extract_zip(location) @@ -673,66 +737,67 @@ def collect_redist_src(location, output, from_inventory, with_structures, zip, q output_location = output copy_list, copy_list_errors = get_copy_list(abouts, location) - copy_errors = copy_redist_src( - copy_list, location, output_location, with_structures) + copy_errors = copy_redist_src(copy_list, location, output_location, with_structures) if zip: import shutil + # Stripped the .zip extension as the `shutil.make_archive` will # append the .zip extension - output_no_extension = output.rsplit('.', 1)[0] - shutil.make_archive(output_no_extension, 'zip', output_location) + output_no_extension = output.rsplit(".", 1)[0] + shutil.make_archive(output_no_extension, "zip", output_location) errors.extend(copy_list_errors) errors.extend(copy_errors) - errors_count = report_errors( - errors, quiet, verbose, log_file_loc=output + '-error.log') + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet: - msg = 'Redistributed sources are copied to {output}.'.format( - **locals()) + msg = "Redistributed sources are copied to {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) + ###################################################################### # check subcommand ###################################################################### # FIXME: This is really only a dupe of the Inventory command -@about.command(cls=AboutCommand, - short_help='Validate that the format of .ABOUT files is correct and report ' - 'errors and warnings.') -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.option('--exclude', - multiple=True, - metavar='PATTERN', - help='Exclude the processing of the specified input pattern (e.g. *tests* or test/).') -@click.option('--license', - is_flag=True, - help='Validate the license_expression value in the input.') -@click.option('--djc', - nargs=2, - type=str, - metavar='api_url api_key', - help='Validate license_expression from a DejaCode License Library ' - 'API URL using the API KEY.') -@click.option('--log', - nargs=1, - metavar='FILE', - help='Path to a file to save the error messages if any.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') + +@about.command( + cls=AboutCommand, + short_help="Validate that the format of .ABOUT files is correct and report " + "errors and warnings.", +) +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.option( + "--exclude", + multiple=True, + metavar="PATTERN", + help="Exclude the processing of the specified input pattern (e.g. *tests* or test/).", +) +@click.option("--license", is_flag=True, help="Validate the license_expression value in the input.") +@click.option( + "--djc", + nargs=2, + type=str, + metavar="api_url api_key", + help="Validate license_expression from a DejaCode License Library API URL using the API KEY.", +) +@click.option( + "--log", nargs=1, metavar="FILE", help="Path to a file to save the error messages if any." +) +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") def check(location, exclude, license, djc, log, verbose): """ -Check .ABOUT file(s) at LOCATION for validity and print error messages. + Check .ABOUT file(s) at LOCATION for validity and print error messages. -LOCATION: Path to an ABOUT file or a directory with ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. """ print_version() @@ -742,13 +807,13 @@ def check(location, exclude, license, djc, log, verbose): if not parent: os.makedirs(parent) - api_url = '' - api_key = '' + api_url = "" + api_key = "" if djc: # Strip the ' and " for api_url, and api_key from input api_url = djc[0].strip("'").strip('"') api_key = djc[1].strip("'").strip('"') - click.echo('Checking ABOUT files...') + click.echo("Checking ABOUT files...") errors, abouts = collect_inventory(location, exclude) @@ -756,14 +821,15 @@ def check(location, exclude, license, djc, log, verbose): if license: from_check = True _key_text_dict, errs = pre_process_and_fetch_license_dict( - abouts, from_check, api_url, api_key) + abouts, from_check, api_url, api_key + ) for e in errs: errors.append(e) - severe_errors_count = report_errors( - errors, quiet=False, verbose=verbose, log_file_loc=log) + severe_errors_count = report_errors(errors, quiet=False, verbose=verbose, log_file_loc=log) sys.exit(severe_errors_count) + ###################################################################### # transform subcommand ###################################################################### @@ -773,56 +839,77 @@ def print_config_help(ctx, param, value): if not value or ctx.resilient_parsing: return from attributecode.transform import tranformer_config_help + click.echo(tranformer_config_help) ctx.exit() -@about.command(cls=AboutCommand, - short_help='Transform a CSV/JSON/XLSX by applying renamings, filters and checks.') -@click.argument('location', - required=True, - callback=partial(validate_extensions, extensions=( - '.csv', '.json', '.xlsx',)), - metavar='LOCATION', - type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True)) -@click.argument('output', - required=True, - callback=partial(validate_extensions, extensions=( - '.csv', '.json', '.xlsx',)), - metavar='OUTPUT', - type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) -@click.option('-c', '--configuration', - metavar='FILE', - type=click.Path(exists=True, dir_okay=False, - readable=True, resolve_path=True), - help='Path to an optional YAML configuration file. See --help-format for ' - 'format help.') -@click.option('--worksheet', - metavar='name', - help='The worksheet name from the INPUT. (Default: the "active" worksheet)') -@click.option('--help-format', - is_flag=True, is_eager=True, expose_value=False, - callback=print_config_help, - help='Show configuration file format help and exit.') -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') -@click.help_option('-h', '--help') +@about.command( + cls=AboutCommand, + short_help="Transform a CSV/JSON/XLSX by applying renamings, filters and checks.", +) +@click.argument( + "location", + required=True, + callback=partial( + validate_extensions, + extensions=( + ".csv", + ".json", + ".xlsx", + ), + ), + metavar="LOCATION", + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), +) +@click.argument( + "output", + required=True, + callback=partial( + validate_extensions, + extensions=( + ".csv", + ".json", + ".xlsx", + ), + ), + metavar="OUTPUT", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), +) +@click.option( + "-c", + "--configuration", + metavar="FILE", + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), + help="Path to an optional YAML configuration file. See --help-format for format help.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option( + "--help-format", + is_flag=True, + is_eager=True, + expose_value=False, + callback=print_config_help, + help="Show configuration file format help and exit.", +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") def transform(location, output, configuration, worksheet, quiet, verbose): # NOQA """ -Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, filters and checks -and then write a new CSV/JSON/XLSX to OUTPUT. + Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, filters and checks + and then write a new CSV/JSON/XLSX to OUTPUT. -LOCATION: Path to a CSV/JSON/XLSX file. + LOCATION: Path to a CSV/JSON/XLSX file. -OUTPUT: Path to CSV/JSON/XLSX inventory file to create. + OUTPUT: Path to CSV/JSON/XLSX inventory file to create. """ - if worksheet and not location.endswith('.xlsx'): - raise click.UsageError( - 'ERROR: --worksheet option only works with .xlsx input.') + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") if not configuration: transformer = Transformer.default() @@ -830,7 +917,7 @@ def transform(location, output, configuration, worksheet, quiet, verbose): # NO transformer = Transformer.from_file(configuration) if not transformer: - msg = 'Cannot transform without Transformer' + msg = "Cannot transform without Transformer" click.echo(msg) sys.exit(1) @@ -838,40 +925,40 @@ def transform(location, output, configuration, worksheet, quiet, verbose): # NO updated_data = [] new_data = [] - if location.endswith('.csv'): + if location.endswith(".csv"): new_data, errors = transform_csv(location) - elif location.endswith('.json'): + elif location.endswith(".json"): new_data, errors = transform_json(location) - elif location.endswith('.xlsx'): + elif location.endswith(".xlsx"): new_data, errors = transform_excel(location, worksheet) if not errors: updated_data, errors = transform_data(new_data, transformer) if not updated_data: - msg = 'The input is empty. Nothing is transformed.' + msg = "The input is empty. Nothing is transformed." click.echo(msg) sys.exit(0) if not errors: - if output.endswith('.csv'): + if output.endswith(".csv"): write_csv(output, updated_data) - elif output.endswith('.json'): + elif output.endswith(".json"): write_json(output, updated_data) else: write_excel(output, updated_data) if not quiet: print_version() - click.echo('Transforming...') + click.echo("Transforming...") - errors_count = report_errors( - errors, quiet, verbose, log_file_loc=output + '-error.log') + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet and not errors: - msg = 'Transformed file is written to {output}.'.format(**locals()) + msg = "Transformed file is written to {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) + ###################################################################### # Error management ###################################################################### @@ -893,8 +980,8 @@ def report_errors(errors, quiet, verbose, log_file_loc=None): for msg in log_msgs: click.echo(msg) if log_msgs and log_file_loc: - with open(log_file_loc, 'w', encoding='utf-8', errors='replace') as lf: - lf.write('\n'.join(log_msgs)) + with open(log_file_loc, "w", encoding="utf-8", errors="replace") as lf: + lf.write("\n".join(log_msgs)) click.echo("Error log: " + log_file_loc) return severe_errors_count @@ -916,17 +1003,17 @@ def get_error_messages(errors, verbose=False): messages = [] if severe_errors: - error_msg = 'Command completed with {} errors or warnings.'.format( - severe_errors_count) + error_msg = "Command completed with {} errors or warnings.".format(severe_errors_count) messages.append(error_msg) for severity, message in severe_errors: - sevcode = severities.get(severity) or 'UNKNOWN' - msg = '{sevcode}: {message}'.format(**locals()) + sevcode = severities.get(severity) or "UNKNOWN" + msg = "{sevcode}: {message}".format(**locals()) messages.append(msg) return messages, severe_errors_count + ###################################################################### # Misc ###################################################################### @@ -945,7 +1032,7 @@ def parse_key_values(key_values): errors = set() parsed_key_values = defaultdict(list) for key_value in key_values: - key, _, value = key_value.partition('=') + key, _, value = key_value.partition("=") key = key.strip().lower() if not key: @@ -962,5 +1049,5 @@ def parse_key_values(key_values): return dict(parsed_key_values), sorted(errors) -if __name__ == '__main__': +if __name__ == "__main__": about() diff --git a/src/attributecode/gen.py b/src/attributecode/gen.py index 3b824c20..a8b2fa54 100644 --- a/src/attributecode/gen.py +++ b/src/attributecode/gen.py @@ -42,7 +42,7 @@ def check_duplicated_columns(location): at location. """ location = add_unc(location) - with open(location, mode='r', encoding='utf-8-sig', errors='replace') as csvfile: + with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile: reader = csv.reader(csvfile) columns = next(reader) columns = [col for col in columns] @@ -62,12 +62,14 @@ def check_duplicated_columns(location): if dupes: dup_msg = [] for name, names in dupes.items(): - names = u', '.join(names) - msg = '%(name)s with %(names)s' % locals() + names = ", ".join(names) + msg = "%(name)s with %(names)s" % locals() dup_msg.append(msg) - dup_msg = u', '.join(dup_msg) - msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() + - 'Please correct the input and re-run.') + dup_msg = ", ".join(dup_msg) + msg = ( + "Duplicated column name(s): %(dup_msg)s\n" % locals() + + "Please correct the input and re-run." + ) err = Error(ERROR, msg) if not err in errors: errors.append(err) @@ -79,10 +81,9 @@ def check_duplicated_about_resource(arp, arp_list): Return error for duplicated about_resource. """ if arp in arp_list: - msg = ("The input has duplicated values in 'about_resource' " - "field: " + arp) + msg = "The input has duplicated values in 'about_resource' field: " + arp return Error(CRITICAL, msg) - return '' + return "" def check_newline_in_file_field(component): @@ -93,13 +94,16 @@ def check_newline_in_file_field(component): for k in component.keys(): if k in file_fields: try: - if '\n' in component[k]: - if k == u'about_resource': + if "\n" in component[k]: + if k == "about_resource": msg = ( - "Multiple lines detected in 'about_resource' for '%s' which is not supported.") % component['about_resource'] + "Multiple lines detected in 'about_resource' for '%s' which is not supported." + ) % component["about_resource"] else: - msg = ("New line character detected in '%s' for '%s' which is not supported." - "\nPlease use ',' to declare multiple files.") % (k, component['about_resource']) + msg = ( + "New line character detected in '%s' for '%s' which is not supported." + "\nPlease use ',' to declare multiple files." + ) % (k, component["about_resource"]) errors.append(Error(CRITICAL, msg)) except: pass @@ -112,13 +116,14 @@ def check_about_resource_filename(arp): empty string if no error is found. """ if invalid_chars(arp): - msg = ("Invalid characters present in 'about_resource' " - "field: " + arp) - return (Error(ERROR, msg)) - return '' + msg = "Invalid characters present in 'about_resource' field: " + arp + return Error(ERROR, msg) + return "" -def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, reference_dir=None, worksheet=None): +def load_inventory( + location, from_attrib=False, base_dir=None, scancode=False, reference_dir=None, worksheet=None +): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects @@ -136,14 +141,14 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r if scancode: inventory = load_scancode_json(location) else: - if location.endswith('.csv'): + if location.endswith(".csv"): dup_cols_err = check_duplicated_columns(location) if dup_cols_err: errors.extend(dup_cols_err) return errors, abouts inventory = load_csv(location) is_spreadsheet = True - elif location.endswith('.xlsx'): + elif location.endswith(".xlsx"): dup_cols_err, inventory = load_excel(location, worksheet) is_spreadsheet = True if dup_cols_err: @@ -163,8 +168,8 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r for component in stripped_inv: if not from_attrib: - if 'about_resource' in component: - arp = component['about_resource'] + if "about_resource" in component: + arp = component["about_resource"] dup_err = check_duplicated_about_resource(arp, arp_list) if dup_err: if not dup_err in errors: @@ -190,17 +195,16 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r for f in required_fields: if f not in fields: - if from_attrib and f == 'about_resource': + if from_attrib and f == "about_resource": continue else: - msg = "Required field: %(f)r not found in the " % locals( - ) + msg = "Required field: %(f)r not found in the " % locals() errors.append(Error(CRITICAL, msg)) return errors, abouts # Set about file path to '' if no 'about_resource' is provided from # the input - if 'about_resource' not in fields: - afp = '' + if "about_resource" not in fields: + afp = "" else: afp = fields.get(model.About.ABOUT_RESOURCE_ATTR) @@ -214,14 +218,14 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r # Update value for 'about_resource' # keep only the filename or '.' if it's a directory - if 'about_resource' in fields: - updated_resource_value = u'' - resource_path = fields['about_resource'] - if resource_path.endswith(u'/'): - updated_resource_value = u'.' + if "about_resource" in fields: + updated_resource_value = "" + resource_path = fields["about_resource"] + if resource_path.endswith("/"): + updated_resource_value = "." else: updated_resource_value = basename(resource_path) - fields['about_resource'] = updated_resource_value + fields["about_resource"] = updated_resource_value ld_errors = about.load_dict( fields, @@ -233,8 +237,8 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r ) for severity, message in ld_errors: - if 'Custom Field' in message: - field_name = message.replace('Custom Field: ', '').strip() + if "Custom Field" in message: + field_name = message.replace("Custom Field: ", "").strip() if not field_name in custom_fields_list: custom_fields_list.append(field_name) else: @@ -242,8 +246,7 @@ def load_inventory(location, from_attrib=False, base_dir=None, scancode=False, r abouts.append(about) if custom_fields_list: - custom_fields_err_msg = 'Field ' + \ - str(custom_fields_list) + ' is a custom field.' + custom_fields_err_msg = "Field " + str(custom_fields_list) + " is a custom field." errors.append(Error(INFO, custom_fields_err_msg)) return errors, abouts @@ -253,14 +256,23 @@ def update_about_resource(self): pass -def generate(location, base_dir, android=None, reference_dir=None, fetch_license=False, fetch_license_djc=False, scancode=False, worksheet=None): +def generate( + location, + base_dir, + android=None, + reference_dir=None, + fetch_license=False, + fetch_license_djc=False, + scancode=False, + worksheet=None, +): """ Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to base_dir. Return errors and about objects. """ notice_dict = {} - api_url = '' - api_key = '' + api_url = "" + api_key = "" gen_license = False # FIXME: use two different arguments: key and url # Check if the fetch_license contains valid argument @@ -281,11 +293,12 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license base_dir=bdir, reference_dir=reference_dir, scancode=scancode, - worksheet=worksheet + worksheet=worksheet, ) if gen_license: license_dict, err = model.pre_process_and_fetch_license_dict( - abouts, api_url=api_url, api_key=api_key) + abouts, api_url=api_url, api_key=api_key + ) if err: for e in err: # Avoid having same error multiple times @@ -295,22 +308,24 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license for about in abouts: # Strip trailing spaces about.about_file_path = about.about_file_path.strip() - if about.about_file_path.startswith('/'): - about.about_file_path = about.about_file_path.lstrip('/') + if about.about_file_path.startswith("/"): + about.about_file_path = about.about_file_path.lstrip("/") # Use the name as the ABOUT file name if about_resource is empty if not about.about_file_path: about.about_file_path = about.name.value - dump_loc = join(bdir, about.about_file_path.lstrip('/')) + dump_loc = join(bdir, about.about_file_path.lstrip("/")) # The following code is to check if there is any directory ends with spaces - split_path = about.about_file_path.split('/') + split_path = about.about_file_path.split("/") dir_endswith_space = False for segment in split_path: - if segment.endswith(' '): - msg = (u'File path : ' - u'%(dump_loc)s ' - u'contains directory name ends with spaces which is not ' - u'allowed. Generation skipped.' % locals()) + if segment.endswith(" "): + msg = ( + "File path : " + "%(dump_loc)s " + "contains directory name ends with spaces which is not " + "allowed. Generation skipped." % locals() + ) errors.append(Error(ERROR, msg)) dir_endswith_space = True break @@ -319,16 +334,26 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license continue try: - licenses_dict = {} if gen_license: # Write generated LICENSE file - license_key_name_context_url_list = about.dump_lic( - dump_loc, license_dict) + license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict) if license_key_name_context_url_list: - for lic_key, lic_name, lic_filename, lic_context, lic_url, spdx_lic_key in license_key_name_context_url_list: + for ( + lic_key, + lic_name, + lic_filename, + lic_context, + lic_url, + spdx_lic_key, + ) in license_key_name_context_url_list: licenses_dict[lic_key] = [ - lic_name, lic_filename, lic_context, lic_url, spdx_lic_key] + lic_name, + lic_filename, + lic_context, + lic_url, + spdx_lic_key, + ] if not lic_name in about.license_name.value: about.license_name.value.append(lic_name) about.license_file.value[lic_filename] = lic_filename @@ -353,12 +378,13 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license follow the standard from Android Open Source Project """ import os + parent_path = os.path.dirname(util.to_posix(dump_loc)) about.android_module_license(parent_path) notice_path, notice_context = about.android_notice(parent_path) if notice_path in notice_dict.keys(): - notice_dict[notice_path] += '\n\n' + notice_context + notice_dict[notice_path] += "\n\n" + notice_context else: notice_dict[notice_path] = notice_context @@ -366,16 +392,14 @@ def generate(location, base_dir, android=None, reference_dir=None, fetch_license # only keep the first 100 char of the exception # TODO: truncated errors are likely making diagnotics harder emsg = repr(e)[:100] - msg = (u'Failed to write .ABOUT file at : ' - u'%(dump_loc)s ' - u'with error: %(emsg)s' % locals()) + msg = "Failed to write .ABOUT file at : %(dump_loc)s with error: %(emsg)s" % locals() errors.append(Error(ERROR, msg)) if android: # Check if there is already a NOTICE file present for path in notice_dict.keys(): if os.path.exists(path): - msg = (u'NOTICE file already exist at: %s' % path) + msg = "NOTICE file already exist at: %s" % path errors.append(Error(ERROR, msg)) else: about.dump_android_notice(path, notice_dict[path]) diff --git a/src/attributecode/licenses.py b/src/attributecode/licenses.py index 9280ab84..393a6fc8 100644 --- a/src/attributecode/licenses.py +++ b/src/attributecode/licenses.py @@ -16,70 +16,70 @@ # Common license keys COMMON_LICENSES = ( - 'aes-128-3.0', - 'agpl-3.0-plus', - 'apache-1.1', - 'apache-2.0', - 'apple-attribution-1997', - 'apple-excl', - 'apsl-2.0', - 'arphic-public', - 'artistic-perl-1.0', - 'artistic-2.0', - 'bitstream', - 'boost-1.0', - 'broadcom-cfe', - 'bsd-new', - 'bsd-original', - 'bsd-original-uc', - 'bsd-simplified', - 'cmu-computing-services', - 'cddl-1.0', - 'cddl-1.1', - 'cpl-1.0', - 'cc-by-2.5', - 'cc-by-sa-3.0', - 'curl', - 'freetype', - 'gpl-1.0-plus', - 'gpl-2.0', - 'gpl-2.0-bison', - 'gpl-2.0-glibc', - 'gpl-2.0-plus', - 'gpl-3.0', - 'gpl-3.0-plus', - 'lgpl-2.0', - 'lgpl-2.0-plus', - 'lgpl-2.1', - 'lgpl-2.1-plus', - 'lgpl-3.0', - 'lgpl-3.0-plus', - 'gpl-2.0-plus-linking', - 'gpl-2.0-broadcom-linking', - 'ijg', - 'isc', - 'larabie', - 'libpng', - 'ms-limited-public', - 'ms-pl', - 'ms-rl', - 'ms-ttf-eula', - 'mit', - 'mpl-1.1', - 'mpl-2.0', - 'net-snmp', - 'npl-1.1', - 'ntpl', - 'openssl-ssleay', - 'ssleay-windows', - 'rsa-md4', - 'rsa-md5', - 'sfl-license', - 'sgi-freeb-2.0', - 'sun-rpc', - 'tcl', - 'tidy', - 'uoi-ncsa', - 'x11', - 'zlib', + "aes-128-3.0", + "agpl-3.0-plus", + "apache-1.1", + "apache-2.0", + "apple-attribution-1997", + "apple-excl", + "apsl-2.0", + "arphic-public", + "artistic-perl-1.0", + "artistic-2.0", + "bitstream", + "boost-1.0", + "broadcom-cfe", + "bsd-new", + "bsd-original", + "bsd-original-uc", + "bsd-simplified", + "cmu-computing-services", + "cddl-1.0", + "cddl-1.1", + "cpl-1.0", + "cc-by-2.5", + "cc-by-sa-3.0", + "curl", + "freetype", + "gpl-1.0-plus", + "gpl-2.0", + "gpl-2.0-bison", + "gpl-2.0-glibc", + "gpl-2.0-plus", + "gpl-3.0", + "gpl-3.0-plus", + "lgpl-2.0", + "lgpl-2.0-plus", + "lgpl-2.1", + "lgpl-2.1-plus", + "lgpl-3.0", + "lgpl-3.0-plus", + "gpl-2.0-plus-linking", + "gpl-2.0-broadcom-linking", + "ijg", + "isc", + "larabie", + "libpng", + "ms-limited-public", + "ms-pl", + "ms-rl", + "ms-ttf-eula", + "mit", + "mpl-1.1", + "mpl-2.0", + "net-snmp", + "npl-1.1", + "ntpl", + "openssl-ssleay", + "ssleay-windows", + "rsa-md4", + "rsa-md5", + "sfl-license", + "sgi-freeb-2.0", + "sun-rpc", + "tcl", + "tidy", + "uoi-ncsa", + "x11", + "zlib", ) diff --git a/src/attributecode/model.py b/src/attributecode/model.py index e935af2b..66c19656 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -97,7 +97,7 @@ def __init__(self, name=None, value=None, required=False, present=False): self.errors = [] def default_value(self): - return '' + return "" def validate(self, *args, **kwargs): """ @@ -110,7 +110,7 @@ def validate(self, *args, **kwargs): if not self.present: # required fields must be present if self.required: - msg = u'Field %(name)s is required' + msg = "Field %(name)s is required" errors.append(Error(CRITICAL, msg % locals())) return errors else: @@ -120,18 +120,17 @@ def validate(self, *args, **kwargs): if not name in boolean_fields and not self.has_content: # ... especially if required if self.required: - msg = u'Field %(name)s is required and empty' + msg = "Field %(name)s is required and empty" severity = CRITICAL else: severity = INFO - msg = u'Field %(name)s is present but empty.' + msg = "Field %(name)s is present but empty." errors.append(Error(severity, msg % locals())) else: # present fields with content go through validation... # first trim any trailing spaces on each line if isinstance(self.original_value, str): - value = '\n'.join(s.rstrip() for s - in self.original_value.splitlines(False)) + value = "\n".join(s.rstrip() for s in self.original_value.splitlines(False)) # then strip leading and trailing spaces value = value.strip() else: @@ -142,7 +141,7 @@ def validate(self, *args, **kwargs): errors.extend(validation_errors) except Exception as e: emsg = repr(e) - msg = u'Error validating field %(name)s: %(value)r: %(emsg)r' + msg = "Error validating field %(name)s: %(value)r: %(emsg)r" errors.append(Error(CRITICAL, msg % locals())) raise @@ -158,14 +157,14 @@ def _validate(self, *args, **kwargs): return [] def _serialized_value(self): - return self.value if self.value else u'' + return self.value if self.value else "" def serialize(self): """ Return a unicode serialization of self in the ABOUT format. """ name = self.name - value = self.serialized_value() or u'' + value = self.serialized_value() or "" if self.has_content or self.value: value = value.splitlines(True) # multi-line @@ -173,29 +172,29 @@ def serialize(self): # This code is used to read the YAML's multi-line format in # ABOUT files # (Test: test_loads_dumps_is_idempotent) - if value[0].strip() == u'|' or value[0].strip() == u'>': - value = u' '.join(value) + if value[0].strip() == "|" or value[0].strip() == ">": + value = " ".join(value) else: # Insert '|' as the indicator for multi-line follow by a # newline character - value.insert(0, u'|\n') + value.insert(0, "|\n") # insert 4 spaces for newline values - value = u' '.join(value) + value = " ".join(value) else: # FIXME: See https://github.com/nexB/aboutcode-toolkit/issues/323 # The yaml.load() will throw error if the parsed value # contains ': ' character. A work around is to put a pipe, '|' # to indicate the whole value as a string - if value and ': ' in value[0]: - value.insert(0, u'|\n') + if value and ": " in value[0]: + value.insert(0, "|\n") # insert 4 spaces for newline values - value = u' '.join(value) + value = " ".join(value) else: - value = u''.join(value) + value = "".join(value) - serialized = u'%(name)s:' % locals() + serialized = "%(name)s:" % locals() if value: - serialized += ' ' + '%(value)s' % locals() + serialized += " " + "%(value)s" % locals() return serialized def serialized_value(self): @@ -203,7 +202,7 @@ def serialized_value(self): Return a unicode serialization of self in the ABOUT format. Does not include a white space for continuations. """ - return self._serialized_value() or u'' + return self._serialized_value() or "" @property def has_content(self): @@ -215,16 +214,18 @@ def __repr__(self): required = self.required has_content = self.has_content present = self.present - r = ('Field(name=%(name)r, value=%(value)r, required=%(required)r, present=%(present)r)') + r = "Field(name=%(name)r, value=%(value)r, required=%(required)r, present=%(present)r)" return r % locals() def __eq__(self, other): """ Equality based on string content value, ignoring spaces. """ - return (isinstance(other, self.__class__) - and self.name == other.name - and self.value == other.value) + return ( + isinstance(other, self.__class__) + and self.name == other.name + and self.value == other.value + ) class StringField(Field): @@ -234,28 +235,34 @@ class StringField(Field): """ def _validate(self, *args, **kwargs): - errors = super(StringField, self)._validate(*args, ** kwargs) + errors = super(StringField, self)._validate(*args, **kwargs) no_special_char_field = [ - 'license_expression', 'license_key', 'license_name', 'declared_license_expression', 'other_license_expression '] + "license_expression", + "license_key", + "license_name", + "declared_license_expression", + "other_license_expression ", + ] name = self.name if name in no_special_char_field: val = self.value special_char = detect_special_char(val) if special_char: - msg = (u'The following character(s) cannot be in the %(name)s: ' - '%(special_char)r' % locals()) + msg = ( + "The following character(s) cannot be in the %(name)s: " + "%(special_char)r" % locals() + ) errors.append(Error(ERROR, msg)) return errors def _serialized_value(self): - return self.value if self.value else u'' + return self.value if self.value else "" def __eq__(self, other): """ Equality based on string content value, ignoring spaces """ - if not (isinstance(other, self.__class__) - and self.name == other.name): + if not (isinstance(other, self.__class__) and self.name == other.name): return False if self.value == other.value: @@ -263,12 +270,12 @@ def __eq__(self, other): # compare values stripped from spaces. Empty and None are equal if self.value: - sval = u''.join(self.value.split()) + sval = "".join(self.value.split()) if not sval: sval = None if other.value: - oval = u''.join(other.value.split()) + oval = "".join(other.value.split()) if not oval: oval = None @@ -283,12 +290,11 @@ class SingleLineField(StringField): """ def _validate(self, *args, **kwargs): - errors = super(SingleLineField, self)._validate(*args, ** kwargs) - if self.value and isinstance(self.value, str) and '\n' in self.value: + errors = super(SingleLineField, self)._validate(*args, **kwargs) + if self.value and isinstance(self.value, str) and "\n" in self.value: name = self.name value = self.original_value - msg = (u'Field %(name)s: Cannot span multiple lines: %(value)s' - % locals()) + msg = "Field %(name)s: Cannot span multiple lines: %(value)s" % locals() errors.append(Error(ERROR, msg)) return errors @@ -303,7 +309,7 @@ def default_value(self): return [] def _validate(self, *args, **kwargs): - errors = super(ListField, self)._validate(*args, ** kwargs) + errors = super(ListField, self)._validate(*args, **kwargs) # reset self.value = [] @@ -320,8 +326,7 @@ def _validate(self, *args, **kwargs): val = val.strip() if not val: name = self.name - msg = (u'Field %(name)s: ignored empty list value' - % locals()) + msg = "Field %(name)s: ignored empty list value" % locals() errors.append(Error(INFO, msg)) continue # keep only unique and report error for duplicates @@ -329,21 +334,19 @@ def _validate(self, *args, **kwargs): self.value.append(val) else: name = self.name - msg = (u'Field %(name)s: ignored duplicated list value: ' - '%(val)r' % locals()) + msg = "Field %(name)s: ignored duplicated list value: %(val)r" % locals() errors.append(Error(WARNING, msg)) return errors def _serialized_value(self): - return self.value if self.value else u'' + return self.value if self.value else "" def __eq__(self, other): """ Equality based on sort-insensitive values """ - if not (isinstance(other, self.__class__) - and self.name == other.name): + if not (isinstance(other, self.__class__) and self.name == other.name): return False if self.value == other.value: @@ -371,11 +374,11 @@ def _validate(self, *args, **kwargs): """ Check that Package URL is valid. Return a list of errors. """ - errors = super(PackageUrlField, self)._validate(*args, ** kwargs) + errors = super(PackageUrlField, self)._validate(*args, **kwargs) name = self.name val = self.value if not self.is_valid_purl(val): - msg = (u'Field %(name)s: Invalid Package URL: %(val)s' % locals()) + msg = "Field %(name)s: Invalid Package URL: %(val)s" % locals() errors.append(Error(WARNING, msg)) return errors @@ -399,12 +402,12 @@ def _validate(self, *args, **kwargs): """ Check that URLs are valid. Return a list of errors. """ - errors = super(UrlListField, self)._validate(*args, ** kwargs) + errors = super(UrlListField, self)._validate(*args, **kwargs) name = self.name val = self.value for url in val: if not self.is_valid_url(url): - msg = (u'Field %(name)s: Invalid URL: %(val)s' % locals()) + msg = "Field %(name)s: Invalid URL: %(val)s" % locals() errors.append(Error(WARNING, msg)) return errors @@ -414,7 +417,7 @@ def is_valid_url(url): Return True if a URL is valid. """ scheme, netloc, _path, _p, _q, _frg = urlparse(url) - valid = scheme in ('http', 'https', 'ftp') and netloc + valid = scheme in ("http", "https", "ftp") and netloc return valid @@ -427,11 +430,11 @@ def _validate(self, *args, **kwargs): """ Check that URL is valid. Return a list of errors. """ - errors = super(UrlField, self)._validate(*args, ** kwargs) + errors = super(UrlField, self)._validate(*args, **kwargs) name = self.name val = self.value if not self.is_valid_url(val): - msg = (u'Field %(name)s: Invalid URL: %(val)s' % locals()) + msg = "Field %(name)s: Invalid URL: %(val)s" % locals() errors.append(Error(WARNING, msg)) return errors @@ -441,7 +444,7 @@ def is_valid_url(url): Return True if a URL is valid. """ scheme, netloc, _path, _p, _q, _frg = urlparse(url) - valid = scheme in ('http', 'https', 'ftp') and netloc + valid = scheme in ("http", "https", "ftp") and netloc return valid @@ -463,11 +466,11 @@ def _validate(self, *args, **kwargs): base_dir is the directory location of the ABOUT file used to resolve relative paths to actual file locations. """ - errors = super(PathField, self)._validate(*args, ** kwargs) - self.about_file_path = kwargs.get('about_file_path') - self.running_inventory = kwargs.get('running_inventory') - self.base_dir = kwargs.get('base_dir') - self.reference_dir = kwargs.get('reference_dir') + errors = super(PathField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") + self.running_inventory = kwargs.get("running_inventory") + self.base_dir = kwargs.get("base_dir") + self.reference_dir = kwargs.get("reference_dir") if self.base_dir: self.base_dir = util.to_posix(self.base_dir) @@ -482,7 +485,7 @@ def _validate(self, *args, **kwargs): paths = {} for path_value in self.value: - p = path_value.split(',') + p = path_value.split(",") for path in p: path = path.strip() path = util.to_posix(path) @@ -490,7 +493,7 @@ def _validate(self, *args, **kwargs): # normalize eventual / to . # and a succession of one or more ////// to . too if path.strip() and not path.strip(posixpath.sep): - path = '.' + path = "." # removing leading and trailing path separator # path are always relative @@ -500,8 +503,10 @@ def _validate(self, *args, **kwargs): # set from the 'license-text-location' option, so the tool should check # at the 'license-text-location' instead of the 'base_dir' if not (self.base_dir or self.reference_dir): - msg = (u'Field %(name)s: Unable to verify path: %(path)s:' - u' No base directory provided' % locals()) + msg = ( + "Field %(name)s: Unable to verify path: %(path)s:" + " No base directory provided" % locals() + ) errors.append(Error(ERROR, msg)) location = None paths[path] = location @@ -519,13 +524,10 @@ def _validate(self, *args, **kwargs): # parent of the 'about_file_path' with the value of the # 'about_resource' arp = posixpath.join(afp_parent, path) - normalized_arp = posixpath.normpath( - arp).strip(posixpath.sep) - location = posixpath.join( - self.base_dir, normalized_arp) + normalized_arp = posixpath.normpath(arp).strip(posixpath.sep) + location = posixpath.join(self.base_dir, normalized_arp) else: - location = posixpath.normpath( - posixpath.join(self.base_dir, path)) + location = posixpath.normpath(posixpath.join(self.base_dir, path)) location = util.to_native(location) location = os.path.abspath(os.path.normpath(location)) @@ -535,10 +537,9 @@ def _validate(self, *args, **kwargs): if not os.path.exists(location): # We don't want to show the UNC_PREFIX in the error message location = util.to_posix(location.strip(UNC_PREFIX)) - msg = (u'Field %(name)s: Path %(location)s not found' - % locals()) + msg = "Field %(name)s: Path %(location)s not found" % locals() # We want to show INFO error for 'about_resource' - if name == u'about_resource': + if name == "about_resource": errors.append(Error(INFO, msg)) else: errors.append(Error(CRITICAL, msg)) @@ -556,12 +557,12 @@ class AboutResourceField(PathField): the paths resolved relative to the about file path. """ - def __init__(self, *args, ** kwargs): - super(AboutResourceField, self).__init__(*args, ** kwargs) + def __init__(self, *args, **kwargs): + super(AboutResourceField, self).__init__(*args, **kwargs) self.resolved_paths = [] def _validate(self, *args, **kwargs): - errors = super(AboutResourceField, self)._validate(*args, ** kwargs) + errors = super(AboutResourceField, self)._validate(*args, **kwargs) return errors @@ -572,12 +573,12 @@ class IgnoredResourcesField(PathField): by the ABOUT file. """ - def __init__(self, *args, ** kwargs): - super(AboutResourceField, self).__init__(*args, ** kwargs) + def __init__(self, *args, **kwargs): + super(AboutResourceField, self).__init__(*args, **kwargs) self.resolved_paths = [] def _validate(self, *args, **kwargs): - errors = super(AboutResourceField, self)._validate(*args, ** kwargs) + errors = super(AboutResourceField, self)._validate(*args, **kwargs) return errors @@ -594,7 +595,7 @@ def _validate(self, *args, **kwargs): of errors. base_dir is the directory used to resolve a file location from a path. """ - errors = super(FileTextField, self)._validate(*args, ** kwargs) + errors = super(FileTextField, self)._validate(*args, **kwargs) # a FileTextField is a PathField # self.value is a paths to location ordered dict # we will replace the location with the text content @@ -608,15 +609,17 @@ def _validate(self, *args, **kwargs): try: # TODO: we have lots the location by replacing it with a text location = add_unc(location) - with open(location, encoding='utf-8', errors='replace') as txt: + with open(location, encoding="utf-8", errors="replace") as txt: text = txt.read() self.value[path] = text except Exception as e: # only keep the first 100 char of the exception emsg = repr(e)[:100] - msg = (u'Field %(name)s: Failed to load text at path: ' - u'%(path)s ' - u'with error: %(emsg)s' % locals()) + msg = ( + "Field %(name)s: Failed to load text at path: " + "%(path)s " + "with error: %(emsg)s" % locals() + ) errors.append(Error(ERROR, msg)) # set or reset self self.errors = errors @@ -631,8 +634,8 @@ class BooleanField(SingleLineField): def default_value(self): return None - true_flags = ('yes', 'y', 'true', 'x') - false_flags = ('no', 'n', 'false') + true_flags = ("yes", "y", "true", "x") + false_flags = ("no", "n", "false") flag_values = true_flags + false_flags def _validate(self, *args, **kwargs): @@ -640,25 +643,27 @@ def _validate(self, *args, **kwargs): Check that flag are valid. Convert flags to booleans. Default flag to False. Return a list of errors. """ - errors = super(BooleanField, self)._validate(*args, ** kwargs) - self.about_file_path = kwargs.get('about_file_path') + errors = super(BooleanField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") flag = self.get_flag(self.original_value) if flag is False: name = self.name val = self.original_value about_file_path = self.about_file_path flag_values = self.flag_values - msg = (u'Path: %(about_file_path)s - Field %(name)s: Invalid flag value: %(val)r is not ' - u'one of: %(flag_values)s' % locals()) + msg = ( + "Path: %(about_file_path)s - Field %(name)s: Invalid flag value: %(val)r is not " + "one of: %(flag_values)s" % locals() + ) errors.append(Error(ERROR, msg)) self.value = None elif flag is None: name = self.name - msg = (u'Field %(name)s: field is present but empty. ' % locals()) + msg = "Field %(name)s: field is present but empty. " % locals() errors.append(Error(INFO, msg)) self.value = None else: - if flag == u'yes' or flag is True: + if flag == "yes" or flag is True: self.value = True else: self.value = False @@ -670,7 +675,7 @@ def get_flag(self, value): possible values or None if empty or False if not found or original value if it is not a boolean value """ - if value is None or value == '': + if value is None or value == "": return None if isinstance(value, bool): @@ -684,9 +689,9 @@ def get_flag(self, value): value = value.lower() if value in self.flag_values: if value in self.true_flags: - return u'yes' + return "yes" else: - return u'no' + return "no" else: return False else: @@ -704,21 +709,23 @@ def has_content(self): def _serialized_value(self): # default normalized values for serialization if self.value: - return u'yes' + return "yes" elif self.value is False: - return u'no' + return "no" else: # self.value is None # TODO: should we serialize to No for None??? - return u'' + return "" def __eq__(self, other): """ Boolean equality """ - return (isinstance(other, self.__class__) - and self.name == other.name - and self.value == other.value) + return ( + isinstance(other, self.__class__) + and self.name == other.name + and self.value == other.value + ) class BooleanAndTwoCharactersField(SingleLineField): @@ -730,8 +737,8 @@ class BooleanAndTwoCharactersField(SingleLineField): def default_value(self): return None - true_flags = ('yes', 'y', 'true', 'x') - false_flags = ('no', 'n', 'false') + true_flags = ("yes", "y", "true", "x") + false_flags = ("no", "n", "false") flag_values = true_flags + false_flags def _validate(self, *args, **kwargs): @@ -739,28 +746,29 @@ def _validate(self, *args, **kwargs): Check that flag are valid with either boolean value or character value. Default flag to False. Return a list of errors. """ - errors = super(BooleanAndTwoCharactersField, - self)._validate(*args, ** kwargs) - self.about_file_path = kwargs.get('about_file_path') + errors = super(BooleanAndTwoCharactersField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") flag = self.get_value(self.original_value) if flag is False: name = self.name val = self.original_value about_file_path = self.about_file_path flag_values = self.flag_values - msg = (u'Path: %(about_file_path)s - Field %(name)s: Invalid value: %(val)r is not ' - u'one of: %(flag_values)s and it is not a 1 or 2 character value.' % locals()) + msg = ( + "Path: %(about_file_path)s - Field %(name)s: Invalid value: %(val)r is not " + "one of: %(flag_values)s and it is not a 1 or 2 character value." % locals() + ) errors.append(Error(ERROR, msg)) self.value = None elif flag is None: name = self.name - msg = (u'Field %(name)s: field is present but empty. ' % locals()) + msg = "Field %(name)s: field is present but empty. " % locals() errors.append(Error(INFO, msg)) self.value = None else: - if flag == u'yes' or flag is True: + if flag == "yes" or flag is True: self.value = True - elif flag == u'no': + elif flag == "no": self.value = False else: self.value = flag @@ -772,7 +780,7 @@ def get_value(self, value): possible values or None if empty or False if not found or original value if it is not a boolean value """ - if value is None or value == '': + if value is None or value == "": return None if isinstance(value, bool): @@ -786,9 +794,9 @@ def get_value(self, value): value = value.lower() if value in self.flag_values or len(value) <= 2: if value in self.true_flags: - return u'yes' + return "yes" elif value in self.false_flags: - return u'no' + return "no" else: return value else: @@ -809,19 +817,18 @@ def _serialized_value(self): # default normalized values for serialization if self.value: if isinstance(self.value, bool): - return u'yes' + return "yes" else: return self.value elif self.value is False: - return u'no' + return "no" else: # self.value is None # TODO: should we serialize to No for None??? - return u'' + return "" -def validate_fields(fields, about_file_path, running_inventory, base_dir, - reference_dir=None): +def validate_fields(fields, about_file_path, running_inventory, base_dir, reference_dir=None): """ Validate a sequence of Field objects. Return a list of errors. Validation may update the Field objects as needed as a side effect. @@ -840,8 +847,10 @@ def validate_fields(fields, about_file_path, running_inventory, base_dir, def validate_field_name(name): if not is_valid_name(name): - msg = ('Field name: %(name)r contains illegal name characters ' - '(or empty spaces) and is ignored.') + msg = ( + "Field name: %(name)r contains illegal name characters " + "(or empty spaces) and is ignored." + ) return Error(WARNING, msg % locals()) @@ -862,20 +871,21 @@ class About(object): """ Represent an ABOUT file and functions to parse and validate a file. """ + # special names, used only when serializing lists of ABOUT files to CSV or # similar # name of the attribute containing the relative ABOUT file path - ABOUT_FILE_PATH_ATTR = 'about_file_path' + ABOUT_FILE_PATH_ATTR = "about_file_path" # name of the attribute containing the resolved relative Resources paths - about_resource_path_attr = 'about_resource_path' + about_resource_path_attr = "about_resource_path" # name of the attribute containing the resolved relative Resources paths - ABOUT_RESOURCE_ATTR = 'about_resource' + ABOUT_RESOURCE_ATTR = "about_resource" # Required fields - required_fields = ['name'] + required_fields = ["name"] def get_required_fields(self): return [f for f in self.fields if f.required] @@ -886,57 +896,52 @@ def set_standard_fields(self): could use a metaclass to track ordering django-like but this approach is simpler. """ - self.fields = dict([ - ('about_resource', AboutResourceField()), - ('ignored_resources', AboutResourceField()), - ('name', SingleLineField(required=True)), - ('version', SingleLineField()), - - ('download_url', UrlField()), - ('description', StringField()), - ('homepage_url', UrlField()), - ('package_url', PackageUrlField()), - ('notes', StringField()), - - ('license_expression', SingleLineField()), - ('license_key', ListField()), - ('license_name', ListField()), - ('license_file', FileTextField()), - ('license_url', UrlListField()), - ('spdx_license_expression', SingleLineField()), - ('spdx_license_key', ListField()), - ('declared_license_expression', SingleLineField()), - ('other_license_expression', SingleLineField()), - ('copyright', StringField()), - ('notice_file', FileTextField()), - ('notice_url', UrlField()), - - ('redistribute', BooleanField()), - ('attribute', BooleanAndTwoCharactersField()), - ('track_changes', BooleanField()), - ('modified', BooleanField()), - ('internal_use_only', BooleanField()), - - ('changelog_file', FileTextField()), - - ('owner', StringField()), - ('owner_url', UrlField()), - ('contact', StringField()), - ('author', StringField()), - ('author_file', FileTextField()), - - ('vcs_tool', SingleLineField()), - ('vcs_repository', SingleLineField()), - ('vcs_path', SingleLineField()), - ('vcs_tag', SingleLineField()), - ('vcs_branch', SingleLineField()), - ('vcs_revision', SingleLineField()), - - ('checksum_md5', SingleLineField()), - ('checksum_sha1', SingleLineField()), - ('checksum_sha256', SingleLineField()), - ('spec_version', SingleLineField()), - ]) + self.fields = dict( + [ + ("about_resource", AboutResourceField()), + ("ignored_resources", AboutResourceField()), + ("name", SingleLineField(required=True)), + ("version", SingleLineField()), + ("download_url", UrlField()), + ("description", StringField()), + ("homepage_url", UrlField()), + ("package_url", PackageUrlField()), + ("notes", StringField()), + ("license_expression", SingleLineField()), + ("license_key", ListField()), + ("license_name", ListField()), + ("license_file", FileTextField()), + ("license_url", UrlListField()), + ("spdx_license_expression", SingleLineField()), + ("spdx_license_key", ListField()), + ("declared_license_expression", SingleLineField()), + ("other_license_expression", SingleLineField()), + ("copyright", StringField()), + ("notice_file", FileTextField()), + ("notice_url", UrlField()), + ("redistribute", BooleanField()), + ("attribute", BooleanAndTwoCharactersField()), + ("track_changes", BooleanField()), + ("modified", BooleanField()), + ("internal_use_only", BooleanField()), + ("changelog_file", FileTextField()), + ("owner", StringField()), + ("owner_url", UrlField()), + ("contact", StringField()), + ("author", StringField()), + ("author_file", FileTextField()), + ("vcs_tool", SingleLineField()), + ("vcs_repository", SingleLineField()), + ("vcs_path", SingleLineField()), + ("vcs_tag", SingleLineField()), + ("vcs_branch", SingleLineField()), + ("vcs_revision", SingleLineField()), + ("checksum_md5", SingleLineField()), + ("checksum_sha1", SingleLineField()), + ("checksum_sha256", SingleLineField()), + ("spec_version", SingleLineField()), + ] + ) for name, field in self.fields.items(): # we could have a hack to get the actual field name @@ -966,7 +971,7 @@ def __init__(self, location=None, about_file_path=None, strict=False): self.base_dir = os.path.dirname(location) self.errors.extend(self.load(location)) if strict and self.errors and filter_errors(self.errors): - msg = '\n'.join(map(str, self.errors)) + msg = "\n".join(map(str, self.errors)) raise Exception(msg) def __repr__(self): @@ -976,9 +981,11 @@ def __eq__(self, other): """ Equality based on fields and custom_fields., i.e. content. """ - return (isinstance(other, self.__class__) - and self.fields == other.fields - and self.custom_fields == other.custom_fields) + return ( + isinstance(other, self.__class__) + and self.fields == other.fields + and self.custom_fields == other.custom_fields + ) def all_fields(self): """ @@ -993,8 +1000,7 @@ def as_dict(self): """ data = {} data[self.ABOUT_FILE_PATH_ATTR] = self.about_file_path - with_values = ((fld.name, fld.serialized_value()) - for fld in self.all_fields()) + with_values = ((fld.name, fld.serialized_value()) for fld in self.all_fields()) non_empty = ((name, value) for name, value in with_values if value) data.update(non_empty) return data @@ -1027,9 +1033,11 @@ def hydrate(self, fields): previous_value = seen_fields.get(name) if previous_value: if value != previous_value: - msg = (u'Field %(orig_name)s is a duplicate. ' - u'Original value: "%(previous_value)s" ' - u'replaced with: "%(value)s"') + msg = ( + "Field %(orig_name)s is a duplicate. " + 'Original value: "%(previous_value)s" ' + 'replaced with: "%(value)s"' + ) errors.append(Error(WARNING, msg % locals())) continue @@ -1050,7 +1058,7 @@ def hydrate(self, fields): illegal_name_list.append(name) continue - msg = 'Custom Field: %(orig_name)s' + msg = "Custom Field: %(orig_name)s" errors.append(Error(INFO, msg % locals())) # is this a known one? custom_field = self.custom_fields.get(name) @@ -1066,21 +1074,30 @@ def hydrate(self, fields): # FIXME: why would this ever fail??? try: if name in dir(self): - raise Exception( - 'Illegal field: %(name)r: %(value)r.' % locals()) + raise Exception("Illegal field: %(name)r: %(value)r." % locals()) setattr(self, name, custom_field) except: - msg = 'Internal error with custom field: %(name)r: %(value)r.' + msg = "Internal error with custom field: %(name)r: %(value)r." errors.append(Error(CRITICAL, msg % locals())) if illegal_name_list: - msg = ('Field name: %(illegal_name_list)r contains illegal name characters ' - '(or empty spaces) and is ignored.') + msg = ( + "Field name: %(illegal_name_list)r contains illegal name characters " + "(or empty spaces) and is ignored." + ) errors.append(Error(WARNING, msg % locals())) return errors - def process(self, fields, about_file_path, running_inventory=False, - base_dir=None, scancode=False, from_attrib=False, reference_dir=None): + def process( + self, + fields, + about_file_path, + running_inventory=False, + base_dir=None, + scancode=False, + from_attrib=False, + reference_dir=None, + ): """ Validate and set as attributes on this About object a sequence of `fields` name/value tuples. Return a list of errors. @@ -1093,8 +1110,7 @@ def process(self, fields, about_file_path, running_inventory=False, # We want to copy the license_files before the validation if reference_dir and not from_attrib: - copy_err = copy_license_notice_files( - fields, base_dir, reference_dir, afp) + copy_err = copy_license_notice_files(fields, base_dir, reference_dir, afp) errors.extend(copy_err) # TODO: why? we validate all fields, not only these hydrated @@ -1108,7 +1124,8 @@ def process(self, fields, about_file_path, running_inventory=False, about_file_path, running_inventory, self.base_dir, - self.reference_dir) + self.reference_dir, + ) errors.extend(validation_errors) return errors @@ -1123,10 +1140,10 @@ def load(self, location): errors = [] try: loc = add_unc(loc) - with open(loc, encoding='utf-8', errors='replace') as txt: + with open(loc, encoding="utf-8", errors="replace") as txt: input_text = txt.read() if not input_text: - msg = 'ABOUT file is empty: %(location)r' + msg = "ABOUT file is empty: %(location)r" errors.append(Error(CRITICAL, msg % locals())) self.errors = errors return errors @@ -1150,15 +1167,14 @@ def load(self, location): """ running_inventory = True data = saneyaml.load(input, allow_duplicate_keys=False) - errs = self.load_dict( - data, base_dir, running_inventory=running_inventory) + errs = self.load_dict(data, base_dir, running_inventory=running_inventory) errors.extend(errs) except Exception as e: # The trace is good for debugging, but probably not good for user to # see the traceback message # trace = traceback.format_exc() # msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s' - msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r' + msg = "Cannot load invalid ABOUT file: %(location)r: %(e)r" errors.append(Error(CRITICAL, msg % locals())) self.errors = errors @@ -1167,7 +1183,15 @@ def load(self, location): # FIXME: should be a from_dict class factory instead # FIXME: running_inventory: remove this : this should be done in the commands, not here - def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, running_inventory=False, reference_dir=None,): + def load_dict( + self, + fields_dict, + base_dir, + scancode=False, + from_attrib=False, + running_inventory=False, + reference_dir=None, + ): """ Load this About object file from a `fields_dict` name/value dict. Return a list of errors. @@ -1180,35 +1204,32 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru for key, value in fields: if not value: continue - if key == u'copyrights': + if key == "copyrights": have_copyright = True - elif key == u'license_detections': + elif key == "license_detections": lic_list = ungroup_licenses_from_sctk(value) lic_exp_list = [] for detected_license in value: - if 'license_expression' in detected_license: - lic_exp_list.append( - detected_license['license_expression']) + if "license_expression" in detected_license: + lic_exp_list.append(detected_license["license_expression"]) if lic_exp_list: - fields.append( - ('license_expression', ' AND '.join(lic_exp_list))) + fields.append(("license_expression", " AND ".join(lic_exp_list))) lic_key_list = [] lic_key_exp_list = [] lic_score_list = [] for lic in lic_list: - _char, lic_keys, _invalid_lic_exp = parse_license_expression( - lic['lic_exp']) + _char, lic_keys, _invalid_lic_exp = parse_license_expression(lic["lic_exp"]) lic_key_list.append(lic_keys) # for lic_key in lic_keys: # lic_key_list.append([lic_key]) for lic in lic_list: - lic_key_exp_list.append(lic['lic_exp']) - lic_score_list.append(lic['score']) - fields.append(('license_key', lic_key_list)) - fields.append(('license_key_expression', lic_key_exp_list)) - fields.append(('license_score', lic_score_list)) + lic_key_exp_list.append(lic["lic_exp"]) + lic_score_list.append(lic["score"]) + fields.append(("license_key", lic_key_list)) + fields.append(("license_key_expression", lic_key_exp_list)) + fields.append(("license_score", lic_score_list)) # The licenses field has been ungrouped and can be removed. # Otherwise, it will gives the following INFO level error @@ -1218,32 +1239,39 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru # Make sure the copyrights is present even is empty to avoid error # when generating with Jinja if not have_copyright: - fields.append(('copyrights', '')) + fields.append(("copyrights", "")) else: for key, value in fields: if not value: # never return empty or absent fields continue - if key == u'licenses': + if key == "licenses": # FIXME: use a license object instead - lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, lic_matched_text = ungroup_licenses( - value) + ( + lic_key, + lic_name, + lic_file, + lic_url, + spdx_lic_key, + lic_score, + lic_matched_text, + ) = ungroup_licenses(value) if lic_key: - fields.append(('license_key', lic_key)) + fields.append(("license_key", lic_key)) if lic_name: - fields.append(('license_name', lic_name)) + fields.append(("license_name", lic_name)) if lic_file: - fields.append(('license_file', lic_file)) + fields.append(("license_file", lic_file)) if lic_url: - fields.append(('license_url', lic_url)) + fields.append(("license_url", lic_url)) if spdx_lic_key: - fields.append(('spdx_license_key', spdx_lic_key)) + fields.append(("spdx_license_key", spdx_lic_key)) # The license score is a key from scancode license scan if lic_score: - fields.append(('license_score', lic_score)) + fields.append(("license_score", lic_score)) if lic_matched_text: - fields.append(('matched_text', lic_matched_text)) + fields.append(("matched_text", lic_matched_text)) # The licenses field has been ungrouped and can be removed. # Otherwise, it will gives the following INFO level error # 'Field licenses is a custom field.' @@ -1263,7 +1291,7 @@ def load_dict(self, fields_dict, base_dir, scancode=False, from_attrib=False, ru return errors @classmethod - def from_dict(cls, about_data, base_dir=''): + def from_dict(cls, about_data, base_dir=""): """ Return an About object loaded from a python dict. """ @@ -1282,28 +1310,33 @@ def dumps(self, licenses_dict=None): license_file = [] license_url = [] spdx_license_key = [] - bool_fields = ['redistribute', 'attribute', - 'track_changes', 'modified', 'internal_use_only'] + bool_fields = [ + "redistribute", + "attribute", + "track_changes", + "modified", + "internal_use_only", + ] for field in self.all_fields(): if not field.value and not field.name in bool_fields: continue - if field.name == 'license_key' and field.value: + if field.name == "license_key" and field.value: license_key = field.value - elif field.name == 'license_name' and field.value: + elif field.name == "license_name" and field.value: license_name = field.value - elif field.name == 'license_file' and field.value: + elif field.name == "license_file" and field.value: # Restore the original_value as it was parsed for # validation purpose if field.original_value: # This line break is for the components that have multiple license # values in CSV format. - if '\n' in field.original_value: - license_file_list = field.original_value.split('\n') + if "\n" in field.original_value: + license_file_list = field.original_value.split("\n") license_file = [] # Strip the carriage return character '\r' See #443 for lic in license_file_list: - if '\r' in lic: - license_file.append(lic.strip('\r')) + if "\r" in lic: + license_file.append(lic.strip("\r")) else: license_file.append(lic) else: @@ -1315,9 +1348,9 @@ def dumps(self, licenses_dict=None): license_file = [field.original_value] else: license_file = list(field.value.keys()) - elif field.name == 'license_url' and field.value: + elif field.name == "license_url" and field.value: license_url = field.value - elif field.name == 'spdx_license_key' and field.value: + elif field.name == "spdx_license_key" and field.value: spdx_license_key = field.value elif field.name in file_fields and field.value: data[field.name] = field.original_value @@ -1328,10 +1361,11 @@ def dumps(self, licenses_dict=None): data[field.name] = field.value # If there is no license_key value, parse the license_expression # and get the parsed license key - if 'license_expression' in data: - if not license_key and data['license_expression']: + if "license_expression" in data: + if not license_key and data["license_expression"]: _spec_char, lic_list, _invalid_lic_exp = parse_license_expression( - data['license_expression']) + data["license_expression"] + ) license_key = lic_list # Group the same license information in a list @@ -1342,17 +1376,16 @@ def dumps(self, licenses_dict=None): for lic_key in license_key: lic_dict = {} if licenses_dict and lic_key in licenses_dict: - lic_dict['key'] = lic_key - lic_name, lic_filename, lic_context, lic_url, spdx_lic_key = licenses_dict[ - lic_key] + lic_dict["key"] = lic_key + lic_name, lic_filename, lic_context, lic_url, spdx_lic_key = licenses_dict[lic_key] if lic_name: - lic_dict['name'] = lic_name + lic_dict["name"] = lic_name if lic_filename: - lic_dict['file'] = lic_filename + lic_dict["file"] = lic_filename if lic_url: - lic_dict['url'] = lic_url + lic_dict["url"] = lic_url if spdx_lic_key: - lic_dict['spdx_license_key'] = spdx_lic_key + lic_dict["spdx_license_key"] = spdx_lic_key # Remove the license information if it has been handled # The following condition is to check if license information @@ -1375,11 +1408,13 @@ def dumps(self, licenses_dict=None): # assume the lic_file (custom license) is referring this specific lic_key # otherwise, the tool shouldn't group them if len(lic_key_copy) == len(license_file): - license_group = list(zip_longest( - lic_key_copy, license_name, license_file, license_url, spdx_license_key)) + license_group = list( + zip_longest(lic_key_copy, license_name, license_file, license_url, spdx_license_key) + ) else: - license_group = list(zip_longest( - lic_key_copy, license_name, [], license_url, spdx_license_key)) + license_group = list( + zip_longest(lic_key_copy, license_name, [], license_url, spdx_license_key) + ) # Add the unhandled_lic_file if any if license_file: for lic_file in license_file: @@ -1388,31 +1423,31 @@ def dumps(self, licenses_dict=None): for lic_group in license_group: lic_dict = {} if lic_group[0]: - lic_dict['key'] = lic_group[0] + lic_dict["key"] = lic_group[0] if lic_group[1]: - lic_dict['name'] = lic_group[1] + lic_dict["name"] = lic_group[1] else: # If no name is given, treat the key as the name if lic_group[0]: - lic_dict['name'] = lic_group[0] + lic_dict["name"] = lic_group[0] if lic_group[2]: - lic_dict['file'] = lic_group[2] + lic_dict["file"] = lic_group[2] if lic_group[3]: - lic_dict['url'] = lic_group[3] + lic_dict["url"] = lic_group[3] if lic_group[4]: - lic_dict['spdx_license_key'] = lic_group[4] + lic_dict["spdx_license_key"] = lic_group[4] lic_dict_list.append(lic_dict) # Format the license information in the same order of the license expression for key in license_key: for lic_dict in lic_dict_list: - if key == lic_dict['key']: - data.setdefault('licenses', []).append(lic_dict) + if key == lic_dict["key"]: + data.setdefault("licenses", []).append(lic_dict) lic_dict_list.remove(lic_dict) break for lic_dict in lic_dict_list: - data.setdefault('licenses', []).append(lic_dict) + data.setdefault("licenses", []).append(lic_dict) return saneyaml.dump(data) @@ -1427,17 +1462,16 @@ def dump(self, location, lic_dict=None): os.makedirs(add_unc(parent)) about_file_path = loc - if not about_file_path.endswith('.ABOUT'): + if not about_file_path.endswith(".ABOUT"): # FIXME: we should not infer some location. - if about_file_path.endswith('/'): - about_file_path = util.to_posix( - os.path.join(parent, os.path.basename(parent))) - about_file_path += '.ABOUT' + if about_file_path.endswith("/"): + about_file_path = util.to_posix(os.path.join(parent, os.path.basename(parent))) + about_file_path += ".ABOUT" if on_windows: about_file_path = add_unc(about_file_path) - with open(about_file_path, mode='w', encoding='utf-8', errors='replace') as dumped: + with open(about_file_path, mode="w", encoding="utf-8", errors="replace") as dumped: dumped.write(genereated_tk_version) dumped.write(self.dumps(lic_dict)) @@ -1448,7 +1482,7 @@ def dump_android_notice(self, path, context): if on_windows: path = add_unc(path) - with open(path, mode='w', encoding='utf-8', errors='replace') as dumped: + with open(path, mode="w", encoding="utf-8", errors="replace") as dumped: dumped.write(context) def android_module_license(self, about_parent_path): @@ -1458,12 +1492,13 @@ def android_module_license(self, about_parent_path): for lic_key in self.license_key.value: # Make uppercase and with dash and spaces and dots replaced by underscore # just to look similar and consistent. - name = 'MODULE_LICENSE_' + \ - lic_key.replace('.', '_').replace( - '-', '_').replace(' ', '_').upper() + name = ( + "MODULE_LICENSE_" + + lic_key.replace(".", "_").replace("-", "_").replace(" ", "_").upper() + ) module_lic_path = os.path.join(about_parent_path, name) # Create an empty MODULE_LICESE_XXX file - open(module_lic_path, 'a').close() + open(module_lic_path, "a").close() def android_notice(self, about_parent_path): """ @@ -1472,8 +1507,8 @@ def android_notice(self, about_parent_path): """ # Create NOTICE file with the combination context of copyright, # notice_file and license_file - notice_path = posixpath.join(about_parent_path, 'NOTICE') - notice_context = '' + notice_path = posixpath.join(about_parent_path, "NOTICE") + notice_context = "" if self.copyright.value: notice_context += self.copyright.value if self.notice_file.value: @@ -1481,13 +1516,13 @@ def android_notice(self, about_parent_path): notice_file_key = notice_file_dict.keys() for key in notice_file_key: if notice_file_dict[key]: - notice_context += '\n' + notice_file_dict[key] + '\n' + notice_context += "\n" + notice_file_dict[key] + "\n" if self.license_file.value: lic_file_dict = self.license_file.value lic_file_key = lic_file_dict.keys() for key in lic_file_key: if lic_file_dict[key]: - notice_context += '\n\n' + lic_file_dict[key] + '\n\n' + notice_context += "\n\n" + lic_file_dict[key] + "\n\n" return notice_path, notice_context def dump_lic(self, location, license_dict): @@ -1495,7 +1530,7 @@ def dump_lic(self, location, license_dict): Write LICENSE files and return the a list of key, name, context and the url as these information are needed for the ABOUT file """ - license_name = license_context = license_url = '' + license_name = license_context = license_url = "" loc = util.to_posix(location) parent = posixpath.dirname(loc) license_key_name_context_url = [] @@ -1506,21 +1541,24 @@ def dump_lic(self, location, license_dict): licenses_list = [] if self.license_expression.present: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - self.license_expression.value) + self.license_expression.value + ) if lic_list: for lic in lic_list: if lic not in licenses_list: licenses_list.append(lic) if self.declared_license_expression.present: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - self.declared_license_expression.value) + self.declared_license_expression.value + ) if lic_list: for lic in lic_list: if lic not in licenses_list: licenses_list.append(lic) if self.other_license_expression.present: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - self.other_license_expression.value) + self.other_license_expression.value + ) if lic_list: for lic in lic_list: if lic not in licenses_list: @@ -1529,26 +1567,45 @@ def dump_lic(self, location, license_dict): self.license_key.value = licenses_list self.license_key.present = True for lic_key in licenses_list: - license_name = '' - license_filename = '' - license_context = '' - license_url = '' - spdx_license_key = '' + license_name = "" + license_filename = "" + license_context = "" + license_url = "" + spdx_license_key = "" if lic_key in license_dict: license_path = posixpath.join(parent, lic_key) - license_path += u'.LICENSE' + license_path += ".LICENSE" license_path = add_unc(license_path) - license_name, license_filename, license_context, license_url, spdx_license_key = license_dict[ - lic_key] - license_info = (lic_key, license_name, license_filename, - license_context, license_url, spdx_license_key) + ( + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) = license_dict[lic_key] + license_info = ( + lic_key, + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) license_key_name_context_url.append(license_info) - with open(license_path, mode='w', encoding='utf-8', newline='\n', errors='replace') as lic: + with open( + license_path, mode="w", encoding="utf-8", newline="\n", errors="replace" + ) as lic: lic.write(license_context) else: # Invalid license issue is already handled - license_info = (lic_key, license_name, license_filename, - license_context, license_url, spdx_license_key) + license_info = ( + lic_key, + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) license_key_name_context_url.append(license_info) return license_key_name_context_url @@ -1571,17 +1628,16 @@ def collect_inventory(location, exclude=None): about_file_path = util.get_relative_path(input_location, about_loc) about = About(about_loc, about_file_path) for severity, message in about.errors: - if 'Custom Field' in message: - field_name = message.replace('Custom Field: ', '').strip() + if "Custom Field" in message: + field_name = message.replace("Custom Field: ", "").strip() if field_name not in custom_fields_list: custom_fields_list.append(field_name) else: - msg = (about_file_path + ": " + message) + msg = about_file_path + ": " + message errors.append(Error(severity, msg)) abouts.append(about) if custom_fields_list: - custom_fields_err_msg = 'Field ' + \ - str(custom_fields_list) + ' is a custom field.' + custom_fields_err_msg = "Field " + str(custom_fields_list) + " is a custom field." errors.append(Error(INFO, custom_fields_err_msg)) return errors, abouts @@ -1600,7 +1656,7 @@ def collect_abouts_license_expression(location): for loc in about_locations: try: loc = add_unc(loc) - with open(loc, encoding='utf-8', errors='replace') as txt: + with open(loc, encoding="utf-8", errors="replace") as txt: input_text = txt.read() # saneyaml.load() will have parsing error if the input has # tab value. Therefore, we should check if the input contains @@ -1608,11 +1664,11 @@ def collect_abouts_license_expression(location): input = replace_tab_with_spaces(input_text) data = saneyaml.load(input, allow_duplicate_keys=False) about = About() - about.load_dict(data, base_dir='') + about.load_dict(data, base_dir="") abouts.append(about) except Exception as e: trace = traceback.format_exc() - msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s' + msg = "Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s" errors.append(Error(CRITICAL, msg % locals())) return errors, abouts @@ -1629,26 +1685,24 @@ def collect_inventory_license_expression(location, scancode=False, worksheet=Non if scancode: inventory = gen.load_scancode_json(location) # ScanCode uses 'detected_license_expression' - if not 'detected_license_expression' in inventory[0]: - errors.append( - Error(CRITICAL, "No 'license_expressions' field in the input.")) + if not "detected_license_expression" in inventory[0]: + errors.append(Error(CRITICAL, "No 'license_expressions' field in the input.")) return errors, abouts else: - if location.endswith('.csv'): + if location.endswith(".csv"): inventory = gen.load_csv(location) - elif location.endswith('.xlsx'): + elif location.endswith(".xlsx"): _dup_cols_err, inventory = gen.load_excel(location, worksheet) else: inventory = gen.load_json(location) # Check if 'license_expression' field is in the input - if not inventory or not 'license_expression' in inventory[0]: - errors.append( - Error(CRITICAL, "No 'license_expression' field in the input.")) + if not inventory or not "license_expression" in inventory[0]: + errors.append(Error(CRITICAL, "No 'license_expression' field in the input.")) return errors, abouts for data in inventory: about = About() - about.load_dict(data, base_dir='', scancode=scancode) + about.load_dict(data, base_dir="", scancode=scancode) abouts.append(about) return errors, abouts @@ -1702,12 +1756,11 @@ def copy_redist_src(copy_list, location, output, with_structure): norm_from_path = norm(from_path) relative_from_path = norm_from_path.partition(util.norm(location))[2] # Need to strip the '/' to use the join - if relative_from_path.startswith('/'): - relative_from_path = relative_from_path.partition('/')[2] + if relative_from_path.startswith("/"): + relative_from_path = relative_from_path.partition("/")[2] # Get the directory name of the output path if with_structure: - output_dir = os.path.dirname(os.path.join( - output, util.norm(relative_from_path))) + output_dir = os.path.dirname(os.path.join(output, util.norm(relative_from_path))) else: output_dir = output err = copy_file(from_path, output_dir) @@ -1737,8 +1790,8 @@ def get_copy_list(abouts, location): if about.redistribute.value: file_exist = True for e in about.errors: - if 'Field about_resource' in e.message and 'not found' in e.message: - msg = e.message + u' and cannot be copied.' + if "Field about_resource" in e.message and "not found" in e.message: + msg = e.message + " and cannot be copied." errors.append(Error(CRITICAL, msg)) file_exist = False continue @@ -1750,8 +1803,7 @@ def get_copy_list(abouts, location): else: norm_from_path = os.path.normpath(from_path) # Get the relative path - relative_from_path = norm_from_path.partition( - util.norm(location))[2] + relative_from_path = norm_from_path.partition(util.norm(location))[2] if os.path.isdir(from_path): if not dir_list: dir_list.append(relative_from_path) @@ -1774,7 +1826,7 @@ def get_copy_list(abouts, location): else: # Check if the file is from "root" # If the file is at root level, it'll add to the copy_list - if not os.path.dirname(relative_from_path) == '/': + if not os.path.dirname(relative_from_path) == "/": file_list.append(relative_from_path) else: copy_list.append(from_path) @@ -1785,16 +1837,16 @@ def get_copy_list(abouts, location): if dir in f: file_list.remove(f) continue - if dir.startswith('/'): - dir = dir.partition('/')[2] + if dir.startswith("/"): + dir = dir.partition("/")[2] absolute_path = os.path.join(location, dir) if on_windows: absolute_path = add_unc(absolute_path) copy_list.append(absolute_path) for f in file_list: - if f.startswith('/'): - f = f.partition('/')[2] + if f.startswith("/"): + f = f.partition("/")[2] absolute_path = os.path.join(location, f) if on_windows: absolute_path = add_unc(absolute_path) @@ -1820,26 +1872,24 @@ def about_object_to_list_of_dictionary(abouts): # TODO: this wholeblock should be under sd_dict() ad = about.as_dict() - if 'about_file_path' in ad.keys(): - afp = ad['about_file_path'] + if "about_file_path" in ad.keys(): + afp = ad["about_file_path"] afp_parent = posixpath.dirname(afp) - afp_parent = '/' + \ - afp_parent if not afp_parent.startswith( - '/') else afp_parent + afp_parent = "/" + afp_parent if not afp_parent.startswith("/") else afp_parent # Update the 'about_resource' field with the relative path # from the output location - if 'about_resource' in ad.keys(): - about_resource = ad['about_resource'] + if "about_resource" in ad.keys(): + about_resource = ad["about_resource"] for resource in about_resource: updated_about_resource = posixpath.normpath( - posixpath.join(afp_parent, resource)) - if resource == u'.': - if not updated_about_resource == '/': - updated_about_resource = updated_about_resource + '/' - ad['about_resource'] = dict( - [(updated_about_resource, None)]) - del ad['about_file_path'] + posixpath.join(afp_parent, resource) + ) + if resource == ".": + if not updated_about_resource == "/": + updated_about_resource = updated_about_resource + "/" + ad["about_resource"] = dict([(updated_about_resource, None)]) + del ad["about_file_path"] serialized.append(ad) return serialized @@ -1850,11 +1900,11 @@ def write_output(abouts, location, format): # NOQA Return a list of Error objects. """ about_dicts = about_object_to_list_of_dictionary(abouts) - if not location == '-': + if not location == "-": location = add_unc(location) - if format == 'csv': + if format == "csv": save_as_csv(location, about_dicts, get_field_names(abouts)) - elif format == 'json': + elif format == "json": save_as_json(location, about_dicts) else: save_as_excel(location, about_dicts) @@ -1865,10 +1915,10 @@ def save_as_json(location, about_dicts): Save the given data as a JSON file or print it to standard output. """ data = util.format_about_dict_for_json_output(about_dicts) - if location == '-': + if location == "-": json.dump(data, sys.stdout, indent=2) else: - with open(location, mode='w') as output_file: + with open(location, mode="w") as output_file: output_file.write(json.dumps(data, indent=2)) @@ -1876,14 +1926,16 @@ def save_as_csv(location, about_dicts, field_names): """ Save the given data as a CSV file or print it to standard output. """ - if location == '-': + if location == "-": writer = csv.DictWriter(sys.stdout, field_names) writer.writeheader() csv_formatted_list = util.format_about_dict_output(about_dicts) for row in csv_formatted_list: writer.writerow(row) else: - with open(location, mode='w', encoding='utf-8', newline='', errors='replace') as output_file: + with open( + location, mode="w", encoding="utf-8", newline="", errors="replace" + ) as output_file: writer = csv.DictWriter(output_file, field_names) writer.writeheader() csv_formatted_list = util.format_about_dict_output(about_dicts) @@ -1899,7 +1951,9 @@ def save_as_excel(location, about_dicts): write_excel(location, formatted_list) -def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, api_key=None, scancode=False, reference=None): +def pre_process_and_fetch_license_dict( + abouts, from_check=False, api_url=None, api_key=None, scancode=False, reference=None +): """ Return a dictionary containing the license information (key, name, text, url) fetched from the ScanCode LicenseDB or DejaCode API. @@ -1909,17 +1963,19 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a errors = [] if api_url: dje_uri = urlparse(api_url) - domain = '{uri.scheme}://{uri.netloc}/'.format(uri=dje_uri) - lic_urn = urljoin(domain, 'urn/?urn=urn:dje:license:') + domain = "{uri.scheme}://{uri.netloc}/".format(uri=dje_uri) + lic_urn = urljoin(domain, "urn/?urn=urn:dje:license:") url = api_url else: - url = 'https://scancode-licensedb.aboutcode.org/' + url = "https://scancode-licensedb.aboutcode.org/" if util.have_network_connection(): if not valid_api_url(url): - msg = u"URL not reachable. Invalid 'URL. License generation is skipped." + msg = "URL not reachable. Invalid 'URL. License generation is skipped." errors.append(Error(ERROR, msg)) else: - msg = u'Network problem. Please check your Internet connection. License generation is skipped.' + msg = ( + "Network problem. Please check your Internet connection. License generation is skipped." + ) errors.append(Error(ERROR, msg)) if errors: @@ -1929,51 +1985,66 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a for about in abouts: # No need to go through all the about objects if '--api_key' is invalid auth_error = Error( - ERROR, u"Authorization denied. Invalid '--api_key'. License generation is skipped.") + ERROR, "Authorization denied. Invalid '--api_key'. License generation is skipped." + ) if auth_error in errors: break if scancode: - lic_exp = '' + lic_exp = "" lic_list = [] if about.detected_license_expression.value: lic_exp = about.detected_license_expression.value about.license_expression.value = lic_exp about.license_expression.present = True - afp = '' + afp = "" if about.about_file_path: afp = about.about_file_path if not about.license_expression.value and about.spdx_license_expression.value: lic_exp_value = "" special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - about.spdx_license_expression.value) + about.spdx_license_expression.value + ) if special_char_in_expression or invalid_lic_exp: if special_char_in_expression: if afp: - msg = (afp + u": The following character(s) cannot be in the spdx_license_expression: " + - str(special_char_in_expression)) + msg = ( + afp + + ": The following character(s) cannot be in the spdx_license_expression: " + + str(special_char_in_expression) + ) else: - msg = (u"The following character(s) cannot be in the spdx_license_expression: " + - str(special_char_in_expression)) + msg = ( + "The following character(s) cannot be in the spdx_license_expression: " + + str(special_char_in_expression) + ) else: if afp: - msg = (afp + u": This spdx_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = ( + afp + + ": This spdx_license_expression is invalid: " + + str(invalid_lic_exp) + ) else: - msg = (u"This spdx_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This spdx_license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) else: spdx_lic_exp_segment = about.spdx_license_expression.value.split() for spdx_lic_key in spdx_lic_exp_segment: if lic_exp_value: - lic_exp_value = lic_exp_value + " " + convert_spdx_expression_to_lic_expression( - spdx_lic_key, spdx_sclickey_dict) + lic_exp_value = ( + lic_exp_value + + " " + + convert_spdx_expression_to_lic_expression( + spdx_lic_key, spdx_sclickey_dict + ) + ) else: lic_exp_value = convert_spdx_expression_to_lic_expression( - spdx_lic_key, spdx_sclickey_dict) + spdx_lic_key, spdx_sclickey_dict + ) if lic_exp_value: about.license_expression.value = lic_exp_value about.license_expression.present = True @@ -1982,66 +2053,80 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a if about.declared_license_expression.value: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - about.declared_license_expression.value) + about.declared_license_expression.value + ) if special_char_in_expression: if afp: - msg = (afp + u": The following character(s) cannot be in the declared_license_expression: " + - str(special_char_in_expression)) + msg = ( + afp + + ": The following character(s) cannot be in the declared_license_expression: " + + str(special_char_in_expression) + ) else: - msg = (u"The following character(s) cannot be in the declared_license_expression: " + - str(special_char_in_expression)) + msg = ( + "The following character(s) cannot be in the declared_license_expression: " + + str(special_char_in_expression) + ) errors.append(Error(ERROR, msg)) if invalid_lic_exp: if afp: - msg = (afp + u": This declared_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = ( + afp + + ": This declared_license_expression is invalid: " + + str(invalid_lic_exp) + ) else: - msg = (u"This declared_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This declared_license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) if lic_list: lic_exp_list.extend(lic_list) if about.other_license_expression.value: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - about.other_license_expression.value) + about.other_license_expression.value + ) if special_char_in_expression: if afp: - msg = (afp + u": The following character(s) cannot be in the other_license_expression: " + - str(special_char_in_expression)) + msg = ( + afp + + ": The following character(s) cannot be in the other_license_expression: " + + str(special_char_in_expression) + ) else: - msg = (u"This declared_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This declared_license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) if invalid_lic_exp: if afp: - msg = (afp + u": This other_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = ( + afp + ": This other_license_expression is invalid: " + str(invalid_lic_exp) + ) else: - msg = (u"This other_license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This other_license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) if lic_list: lic_exp_list.extend(lic_list) if about.license_expression.value: special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( - about.license_expression.value) + about.license_expression.value + ) if special_char_in_expression: if afp: - msg = (afp + u": The following character(s) cannot be in the license_expression: " + - str(special_char_in_expression)) + msg = ( + afp + + ": The following character(s) cannot be in the license_expression: " + + str(special_char_in_expression) + ) else: - msg = (u"The following character(s) cannot be in the license_expression: " + - str(special_char_in_expression)) + msg = "The following character(s) cannot be in the license_expression: " + str( + special_char_in_expression + ) errors.append(Error(ERROR, msg)) if invalid_lic_exp: if afp: - msg = (afp + u": This license_expression is invalid: " + - str(invalid_lic_exp)) + msg = afp + ": This license_expression is invalid: " + str(invalid_lic_exp) else: - msg = (u"This license_expression is invalid: " + - str(invalid_lic_exp)) + msg = "This license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) if lic_list: lic_exp_list.extend(lic_list) @@ -2051,16 +2136,15 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a if lic_exp_list: for lic_key in lic_exp_list: if not lic_key in captured_license: - lic_url = '' - license_name = '' - license_filename = '' - license_text = '' - spdx_license_key = '' + lic_url = "" + license_name = "" + license_filename = "" + license_text = "" + spdx_license_key = "" detail_list = [] captured_license.append(lic_key) if api_key: - license_data, errs = api.get_license_details_from_api( - url, api_key, lic_key) + license_data, errs = api.get_license_details_from_api(url, api_key, lic_key) # Catch incorrect API URL if errs: _, msg = errs[0] @@ -2068,7 +2152,7 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a errors.extend(errs) return key_text_dict, errors for severity, message in errs: - msg = (afp + ": " + message) + msg = afp + ": " + message errors.append(Error(severity, msg)) # We don't want to actually get the license information from the # check utility @@ -2076,36 +2160,33 @@ def pre_process_and_fetch_license_dict(abouts, from_check=False, api_url=None, a continue if not license_data: continue - license_name = license_data.get('short_name', '') - license_text = license_data.get('full_text', '') - spdx_license_key = license_data.get( - 'spdx_license_key', '') - license_filename = lic_key + '.LICENSE' + license_name = license_data.get("short_name", "") + license_text = license_data.get("full_text", "") + spdx_license_key = license_data.get("spdx_license_key", "") + license_filename = lic_key + ".LICENSE" lic_url = lic_urn + lic_key else: - license_url = url + lic_key + '.json' - license_text_url = url + lic_key + '.LICENSE' + license_url = url + lic_key + ".json" + license_text_url = url + lic_key + ".LICENSE" try: response = head(license_url) if response.status_code < 400: - json_url_content = get( - license_url).text + json_url_content = get(license_url).text # We don't want to actually get the license # information from the check utility if from_check: continue data = json.loads(json_url_content) - license_name = data['short_name'] - license_text = get( - license_text_url).text - license_filename = data['key'] + '.LICENSE' + license_name = data["short_name"] + license_text = get(license_text_url).text + license_filename = data["key"] + ".LICENSE" lic_url = url + license_filename - spdx_license_key = data['spdx_license_key'] + spdx_license_key = data["spdx_license_key"] else: if afp: - msg = afp + u" : Invalid 'license': " + lic_key + msg = afp + " : Invalid 'license': " + lic_key else: - msg = u"Invalid 'license': " + lic_key + msg = "Invalid 'license': " + lic_key errors.append(Error(ERROR, msg)) continue except exceptions.RequestException as e: @@ -2130,15 +2211,12 @@ def convert_spdx_expression_to_lic_expression(spdx_key, spdx_lic_dict): if spdx_key in spdx_lic_dict: value = spdx_lic_dict[spdx_key] else: - if spdx_key.startswith('('): - mod_key = spdx_key.partition('(')[2] - value = '(' + \ - convert_spdx_expression_to_lic_expression( - mod_key, spdx_lic_dict) - elif spdx_key.endswith(')'): - mod_key = spdx_key.rpartition(')')[0] - value = convert_spdx_expression_to_lic_expression( - mod_key, spdx_lic_dict) + ')' + if spdx_key.startswith("("): + mod_key = spdx_key.partition("(")[2] + value = "(" + convert_spdx_expression_to_lic_expression(mod_key, spdx_lic_dict) + elif spdx_key.endswith(")"): + mod_key = spdx_key.rpartition(")")[0] + value = convert_spdx_expression_to_lic_expression(mod_key, spdx_lic_dict) + ")" else: # This can be operator or key that don't have match value = spdx_key @@ -2148,7 +2226,7 @@ def convert_spdx_expression_to_lic_expression(spdx_key, spdx_lic_dict): def parse_license_expression(lic_expression): licensing = Licensing() lic_list = [] - invalid_lic_exp = '' + invalid_lic_exp = "" special_char = detect_special_char(lic_expression) if not special_char: # Parse the license expression and save it into a list @@ -2161,8 +2239,28 @@ def parse_license_expression(lic_expression): def detect_special_char(expression): not_support_char = [ - '!', '@', '#', '$', '^', '&', '*', '=', '{', '}', - '|', '[', ']', '\\', ':', ';', '<', '>', '?', ',', '/'] + "!", + "@", + "#", + "$", + "^", + "&", + "*", + "=", + "{", + "}", + "|", + "[", + "]", + "\\", + ":", + ";", + "<", + ">", + "?", + ",", + "/", + ] special_character = [] for char in not_support_char: if char in expression: diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index 46b2412e..f0972f71 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -40,7 +40,7 @@ def transform_csv(location): dupes = check_duplicate_fields(field_names) if dupes: - msg = u'Duplicated field name: %(name)s' + msg = "Duplicated field name: %(name)s" for name in dupes: errors.append(Error(CRITICAL, msg % locals())) @@ -72,7 +72,7 @@ def transform_excel(location, worksheet=None): new_data = [] dupes, new_data = read_excel(location, worksheet) if dupes: - msg = u'Duplicated field name: %(name)s' + msg = "Duplicated field name: %(name)s" for name in dupes: errors.append(Error(CRITICAL, msg % locals())) return new_data, errors @@ -110,7 +110,7 @@ def normalize_dict_data(data): """ try: # Check if this is a JSON output from scancode-toolkit - if (data["headers"][0]["tool_name"] == "scancode-toolkit"): + if data["headers"][0]["tool_name"] == "scancode-toolkit": # only takes data inside "files" new_data = data["files"] except: @@ -130,12 +130,10 @@ def transform_data(data, transformer): renamed_field_data = transformer.apply_renamings(data) if transformer.field_filters: - renamed_field_data = list( - transformer.filter_fields(renamed_field_data)) + renamed_field_data = list(transformer.filter_fields(renamed_field_data)) if transformer.exclude_fields: - renamed_field_data = list( - transformer.filter_excluded(renamed_field_data)) + renamed_field_data = list(transformer.filter_excluded(renamed_field_data)) errors = transformer.check_required_fields(renamed_field_data) if errors: @@ -143,7 +141,7 @@ def transform_data(data, transformer): return renamed_field_data, errors -tranformer_config_help = ''' +tranformer_config_help = """ A transform configuration file is used to describe which transformations and validations to apply to a source CSV file. This is a simple text file using YAML format, using the same format as an .ABOUT file. @@ -201,7 +199,7 @@ def transform_data(data, transformer): exclude_fields: - type - temp -''' +""" @attr.attributes @@ -222,6 +220,7 @@ class Transformer(object): # called by attr after the __init__() def __attrs_post_init__(self, *args, **kwargs): from attributecode.model import About + about = About() self.essential_fields = list(about.required_fields) self.standard_fields = [f.name for f in about.all_fields()] @@ -244,13 +243,13 @@ def from_file(cls, location): Load and return a Transformer instance from a YAML configuration file at `location`. """ - with open(location, encoding='utf-8', errors='replace') as conf: + with open(location, encoding="utf-8", errors="replace") as conf: data = saneyaml.load(replace_tab_with_spaces(conf.read())) return cls( - field_renamings=data.get('field_renamings', {}), - required_fields=data.get('required_fields', []), - field_filters=data.get('field_filters', []), - exclude_fields=data.get('exclude_fields', []), + field_renamings=data.get("field_renamings", {}), + required_fields=data.get("required_fields", []), + field_filters=data.get("field_filters", []), + exclude_fields=data.get("exclude_fields", []), ) def check_required_fields(self, data): @@ -268,8 +267,8 @@ def check_required_fields(self, data): if not missings: continue - missings = ', '.join(missings) - msg = 'Row {rn} is missing required values for fields: {missings}' + missings = ", ".join(missings) + msg = "Row {rn} is missing required values for fields: {missings}" errors.append(Error(CRITICAL, msg.format(**locals()))) return errors @@ -291,8 +290,7 @@ def apply_renamings(self, data): for idx, renamed_from_key in enumerate(renamed_from_list): if key == renamed_from_key: renamed_key = renamed_to_list[idx] - renamed_obj[renamed_key] = self.apply_renamings( - value) + renamed_obj[renamed_key] = self.apply_renamings(value) else: renamed_obj[key] = self.apply_renamings(value) return renamed_obj @@ -357,7 +355,7 @@ def read_csv_rows(location): """ Yield rows (as a list of values) from a CSV file at `location`. """ - with open(location, encoding='utf-8', errors='replace') as csvfile: + with open(location, encoding="utf-8", errors="replace") as csvfile: reader = csv.reader(csvfile) for row in reader: yield row @@ -367,7 +365,7 @@ def read_json(location): """ Yield rows (as a list of values) from a CSV file at `location`. """ - with open(location, encoding='utf-8', errors='replace') as jsonfile: + with open(location, encoding="utf-8", errors="replace") as jsonfile: return json.load(jsonfile) @@ -376,7 +374,7 @@ def write_csv(location, data): Write a CSV file at `location` with the `data` which is a list of ordered dicts. """ field_names = list(data[0].keys()) - with open(location, 'w', encoding='utf-8', newline='\n', errors='replace') as csvfile: + with open(location, "w", encoding="utf-8", newline="\n", errors="replace") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=field_names) writer.writeheader() writer.writerows(data) @@ -386,7 +384,7 @@ def write_json(location, data): """ Write a JSON file at `location` the `data` list of ordered dicts. """ - with open(location, 'w') as jsonfile: + with open(location, "w") as jsonfile: json.dump(data, jsonfile, indent=3) @@ -410,7 +408,7 @@ def read_excel(location, worksheet=None): while index <= max_col: value = sheet_obj.cell(row=1, column=index).value if value in col_keys: - msg = 'Duplicated column name, ' + str(value) + ', detected.' + msg = "Duplicated column name, " + str(value) + ", detected." errors.append(Error(CRITICAL, msg)) return errors, results if value in mapping_dict: @@ -426,7 +424,7 @@ def read_excel(location, worksheet=None): if value: row_dict[col_keys[index]] = value else: - row_dict[col_keys[index]] = '' + row_dict[col_keys[index]] = "" index = index + 1 results.append(row_dict) return errors, results diff --git a/src/attributecode/util.py b/src/attributecode/util.py index 5f398d7d..d63e6b6a 100644 --- a/src/attributecode/util.py +++ b/src/attributecode/util.py @@ -47,7 +47,7 @@ def to_posix(path): - """ + r""" Return a path using the posix path separator given a path that may contain posix or windows separators, converting "\\" to "/". NB: this path will still be valid in the windows explorer (except for a UNC or share name). It @@ -65,9 +65,7 @@ def to_posix(path): ) valid_file_chars = "_-.+()~[]{}@%!$," -invalid_file_chars = string.punctuation.translate( - str.maketrans("", "", valid_file_chars) -) +invalid_file_chars = string.punctuation.translate(str.maketrans("", "", valid_file_chars)) def invalid_chars(path): @@ -167,7 +165,8 @@ def get_locations(location): """ location = add_unc(location) location = get_absolute(location) - assert os.path.exists(location) + if not os.path.exists(location): + raise FileNotFoundError(f"Expected path does not exist: {location}") if os.path.isfile(location): yield location @@ -283,9 +282,8 @@ def get_relative_path(base_loc, full_loc): base = norm(base_loc) path = norm(full_loc) - assert path.startswith(base), ( - "Cannot compute relative path: %(path)r does not start with %(base)r" % locals() - ) + if not path.startswith(base): + raise ValueError(f"Cannot compute relative path: {path!r} does not start with {base!r}") base_name = resource_name(base) no_dir = base == base_name same_loc = base == path @@ -308,7 +306,7 @@ def get_relative_path(base_loc, full_loc): def to_native(path): - """ + r""" Return a path using the current OS path separator given a path that may contain posix or windows separators, converting "/" to "\\" on windows and "\\" to "/" on posix OSes. @@ -526,9 +524,7 @@ def copy_file(from_path, to_path): file_name = os.path.basename(from_path) to_file_path = os.path.join(to_path, file_name) if os.path.exists(to_file_path): - msg = ( - to_file_path + " is already existed and is replaced by " + from_path - ) + msg = to_file_path + " is already existed and is replaced by " + from_path error = Error(WARNING, msg) shutil.copy2(from_path, to_path) return error @@ -637,9 +633,7 @@ def format_about_dict_for_json_output(about_dictionary_list): row_list[key] = element[key] # Group the same license information in a list - license_group = list( - zip_longest(license_key, license_name, license_file, license_url) - ) + license_group = list(zip_longest(license_key, license_name, license_file, license_url)) if license_group: licenses_list = [] for lic_group in license_group: @@ -732,9 +726,7 @@ def get_file_text(file_name, reference): msg = "The file " + file_path + " does not exist" error = Error(CRITICAL, msg) else: - with codecs.open( - file_path, "rb", encoding="utf-8-sig", errors="replace" - ) as txt: + with codecs.open(file_path, "rb", encoding="utf-8-sig", errors="replace") as txt: # with io.open(file_path, encoding='utf-8') as txt: text = txt.read() return error, text @@ -855,8 +847,8 @@ def write_licenses(lic_dict, location): def strip_inventory_value(inventory): """ - The inventory is a list of dictionaries. This function will strip the value - of the dictionary and return the stripped dictionary to a list + Strip the value of the dictionary and return the stripped dictionary to + a list. """ stripped_inventory = [] for component in inventory: diff --git a/tests/test_api.py b/tests/test_api.py index 156fa3b1..6597ffdf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -35,53 +35,58 @@ def read(self): class ApiTest(unittest.TestCase): - - @mock.patch.object(api, 'request_license_data') + @mock.patch.object(api, "request_license_data") def test_api_get_license_details_from_api(self, request_license_data): license_data = { - 'short_name': 'Apache 2.0', - 'full_text': 'Apache License Version 2.0 ...', - 'key': 'apache-2.0', + "short_name": "Apache 2.0", + "full_text": "Apache License Version 2.0 ...", + "key": "apache-2.0", } errors = [] request_license_data.return_value = license_data, errors - expected = ({'short_name': 'Apache 2.0', - 'full_text': 'Apache License Version 2.0 ...', 'key': 'apache-2.0'}, []) + expected = ( + { + "short_name": "Apache 2.0", + "full_text": "Apache License Version 2.0 ...", + "key": "apache-2.0", + }, + [], + ) result = api.get_license_details_from_api( - api_url='api_url', api_key='api_key', license_key='license_key') + api_url="api_url", api_key="api_key", license_key="license_key" + ) assert expected == result - @mock.patch.object(api, 'get') + @mock.patch.object(api, "get") def test_api_request_license_data_with_result(self, mock_data): response_content = ( b'{"count":1,"results":[{"name":"Apache 2.0","key":"apache-2.0","text":"Text"}]}' ) mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data( - api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') - expected = ( - {'name': 'Apache 2.0', 'key': 'apache-2.0', 'text': 'Text'}, - [] + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" ) + expected = ({"name": "Apache 2.0", "key": "apache-2.0", "text": "Text"}, []) assert expected == license_data - @mock.patch.object(api, 'get') + @mock.patch.object(api, "get") def test_api_request_license_data_without_result(self, mock_data): response_content = b'{"count":0,"results":[]}' mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data( - api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" + ) expected = ({}, [Error(ERROR, "Invalid 'license': apache-2.0")]) assert expected == license_data - @mock.patch.object(api, 'get') + @mock.patch.object(api, "get") def test_api_request_license_data_with_incorrect_url(self, mock_data): # Some URL that is accessible but not a correct API URL - response_content = b'' + response_content = b"" mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data( - api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') - expected = ( - {}, [Error(ERROR, "Invalid '--api_url'. License generation is skipped.")]) + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" + ) + expected = ({}, [Error(ERROR, "Invalid '--api_url'. License generation is skipped.")]) assert expected == license_data diff --git a/tests/test_attrib.py b/tests/test_attrib.py index 59b7d919..da7c69e1 100644 --- a/tests/test_attrib.py +++ b/tests/test_attrib.py @@ -28,42 +28,41 @@ class TemplateTest(unittest.TestCase): - def test_check_template_simple_valid_returns_None(self): expected = None - assert expected == attrib.check_template('template_string') + assert expected == attrib.check_template("template_string") def test_check_template_complex_valid_returns_None(self): - template = ''' + template = """ {% for about in abouts -%} {{ about.name.value }}: {{ about.version.value }} {% for res in about.about_resource.value -%} resource: {{ res }} {% endfor -%} - {% endfor -%}''' + {% endfor -%}""" expected = None assert expected == attrib.check_template(template) def test_check_template_complex_invalid_returns_error(self): - template = ''' + template = """ {% for about in abouts -%} {{ about.name.value }}: {{ about.version.value }} {% for res in about.about_ressdsdsdsdsdsdource.value -%} resource: {{] res }} {% endfor -%} - {% endfor -%}''' + {% endfor -%}""" expected = (5, "unexpected ']'") assert expected == attrib.check_template(template) def test_check_template_invalid_return_error_lineno_and_message(self): expected = 1, "unexpected end of template, expected 'end of print statement'." - assert expected == attrib.check_template('{{template_string') + assert expected == attrib.check_template("{{template_string") def test_check_template_all_builtin_templates_are_valid(self): builtin_templates_dir = os.path.dirname(attrib.DEFAULT_TEMPLATE_FILE) for template in os.listdir(builtin_templates_dir): template_loc = os.path.join(builtin_templates_dir, template) - with open(template_loc, 'r', encoding='utf-8', errors='replace') as tmpl: + with open(template_loc, "r", encoding="utf-8", errors="replace") as tmpl: template = tmpl.read() try: assert None == attrib.check_template(template) @@ -72,32 +71,29 @@ def test_check_template_all_builtin_templates_are_valid(self): class GenerateTest(unittest.TestCase): - def test_generate_from_collected_inventory_wih_custom_temaplte(self): - test_file = get_test_loc('test_attrib/gen_simple/attrib.ABOUT') + test_file = get_test_loc("test_attrib/gen_simple/attrib.ABOUT") errors, abouts = model.collect_inventory(test_file) assert not errors - test_template = get_test_loc('test_attrib/gen_simple/test.template') + test_template = get_test_loc("test_attrib/gen_simple/test.template") with open(test_template) as tmpl: template = tmpl.read() - expected = ( - 'Apache HTTP Server: 2.4.3\n' - 'resource: httpd-2.4.3.tar.gz\n') + expected = "Apache HTTP Server: 2.4.3\nresource: httpd-2.4.3.tar.gz\n" license_dict = {} is_about_input = True min_license_score = 0 scancode = False error, result = attrib.generate( - abouts, is_about_input, license_dict, scancode, min_license_score, template=template) + abouts, is_about_input, license_dict, scancode, min_license_score, template=template + ) assert expected == result assert not error def test_generate_with_default_template(self): - test_file = get_test_loc( - 'test_attrib/gen_default_template/attrib.ABOUT') + test_file = get_test_loc("test_attrib/gen_default_template/attrib.ABOUT") errors, abouts = model.collect_inventory(test_file) assert not errors @@ -107,11 +103,13 @@ def test_generate_with_default_template(self): scancode = False error, result = attrib.generate_from_file( - abouts, is_about_input, license_dict, scancode, min_license_score) + abouts, is_about_input, license_dict, scancode, min_license_score + ) assert not error expected_file = get_test_loc( - 'test_attrib/gen_default_template/expected_default_attrib.html') + "test_attrib/gen_default_template/expected_default_attrib.html" + ) with open(expected_file) as exp: expected = exp.read() @@ -119,17 +117,14 @@ def test_generate_with_default_template(self): result = remove_timestamp(result) expected = remove_timestamp(expected) # Ignore all white spaces and newline - result = result.replace('\n', '').replace(' ', '') - expected = expected.replace('\n', '').replace(' ', '') + result = result.replace("\n", "").replace(" ", "") + expected = expected.replace("\n", "").replace(" ", "") assert expected == result def test_lic_key_name_sync(self): - test_file = get_test_loc( - 'test_attrib/gen_license_key_name_check/test.ABOUT') - expected = get_test_loc( - 'test_attrib/gen_license_key_name_check/expected/expected.html') - template_loc = get_test_loc( - 'test_attrib/gen_license_key_name_check/custom.template') + test_file = get_test_loc("test_attrib/gen_license_key_name_check/test.ABOUT") + expected = get_test_loc("test_attrib/gen_license_key_name_check/expected/expected.html") + template_loc = get_test_loc("test_attrib/gen_license_key_name_check/custom.template") output_file = get_temp_file() license_dict = {} @@ -137,7 +132,8 @@ def test_lic_key_name_sync(self): errors, abouts = model.collect_inventory(test_file) attrib.generate_and_save( - abouts, is_about_input, license_dict, output_file, template_loc=template_loc) + abouts, is_about_input, license_dict, output_file, template_loc=template_loc + ) with open(output_file) as of: f1 = [line.strip() for line in of if line.strip()] @@ -147,8 +143,7 @@ def test_lic_key_name_sync(self): assert f1 == f2 def test_scancode_input_min_score_0(self): - test_file = get_test_loc( - 'test_attrib/scancode_input/sc-2-licenses.json') + test_file = get_test_loc("test_attrib/scancode_input/sc-2-licenses.json") errors, abouts = gen.load_inventory(test_file, scancode=True) # Check if there is error's level > INFO result = [(level, e) for level, e in errors if level > INFO] @@ -157,14 +152,13 @@ def test_scancode_input_min_score_0(self): is_about_input = False scancode = True - lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict( - abouts) + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) errors, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=0) + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) assert not errors - expected_file = get_test_loc( - 'test_attrib/scancode_input/sc-min_score-0.html') + expected_file = get_test_loc("test_attrib/scancode_input/sc-min_score-0.html") with open(expected_file) as exp: expected = exp.read() @@ -175,12 +169,12 @@ def test_scancode_input_min_score_0(self): # expected doesn't work well, it works after removed all the newline and spaces # assert expected == result # assert expected.splitlines(False) == result.splitlines(False) - assert expected.replace('\n', '').replace(' ', '').replace( - '\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '') + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") def test_scancode_input_min_score_100(self): - test_file = get_test_loc( - 'test_attrib/scancode_input/sc-2-licenses.json') + test_file = get_test_loc("test_attrib/scancode_input/sc-2-licenses.json") errors, abouts = gen.load_inventory(test_file, scancode=True) # Check if there is error's level > INFO result = [(level, e) for level, e in errors if level > INFO] @@ -189,14 +183,13 @@ def test_scancode_input_min_score_100(self): is_about_input = False scancode = True - lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict( - abouts) + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) errors, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=100) + abouts, is_about_input, lic_dict, scancode, min_license_score=100 + ) assert not errors - expected_file = get_test_loc( - 'test_attrib/scancode_input/sc.html') + expected_file = get_test_loc("test_attrib/scancode_input/sc.html") with open(expected_file) as exp: expected = exp.read() @@ -207,11 +200,12 @@ def test_scancode_input_min_score_100(self): # expected doesn't work well, it works after removed all the newline and spaces # assert expected == result # assert expected.splitlines(False) == result.splitlines(False) - assert expected.replace('\n', '').replace(' ', '').replace( - '\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '') + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") def test_scancode_input_dup_lic(self): - test_file = get_test_loc('test_attrib/scancode_input/sc-dup-lic.json') + test_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic.json") errors, abouts = gen.load_inventory(test_file, scancode=True) # Check if there is error's level > INFO result = [(level, e) for level, e in errors if level > INFO] @@ -220,14 +214,13 @@ def test_scancode_input_dup_lic(self): is_about_input = False scancode = True - lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict( - abouts) + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) errors, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=0) + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) assert not errors - expected_file = get_test_loc( - 'test_attrib/scancode_input/sc-dup-lic.html') + expected_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic.html") with open(expected_file) as exp: expected = exp.read() @@ -238,12 +231,12 @@ def test_scancode_input_dup_lic(self): # expected doesn't work well, it works after removed all the newline and spaces # assert expected == result # assert expected.splitlines(False) == result.splitlines(False) - assert expected.replace('\n', '').replace(' ', '').replace( - '\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '') + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") def test_scancode_input_dup_lic_match(self): - test_file = get_test_loc( - 'test_attrib/scancode_input/sc-dup-lic-match.json') + test_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic-match.json") errors, abouts = gen.load_inventory(test_file, scancode=True) print("############################") print(errors) @@ -254,14 +247,13 @@ def test_scancode_input_dup_lic_match(self): is_about_input = False scancode = True - lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict( - abouts) + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) errors, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=0) + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) assert not errors - expected_file = get_test_loc( - 'test_attrib/scancode_input/sc-dup-lic-match.html') + expected_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic-match.html") with open(expected_file) as exp: expected = exp.read() @@ -272,12 +264,12 @@ def test_scancode_input_dup_lic_match(self): # expected doesn't work well, it works after removed all the newline and spaces # assert expected == result # assert expected.splitlines(False) == result.splitlines(False) - assert expected.replace('\n', '').replace(' ', '').replace( - '\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '') + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") def test_scancode_input_multi_lic(self): - test_file = get_test_loc( - 'test_attrib/scancode_input/sc-multi-lic.json') + test_file = get_test_loc("test_attrib/scancode_input/sc-multi-lic.json") errors, abouts = gen.load_inventory(test_file, scancode=True) # Check if there is error's level > INFO result = [(level, e) for level, e in errors if level > INFO] @@ -286,14 +278,13 @@ def test_scancode_input_multi_lic(self): is_about_input = False scancode = True - lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict( - abouts) + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) errors, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=0) + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) assert not errors - expected_file = get_test_loc( - 'test_attrib/scancode_input/sc-multi-lic.html') + expected_file = get_test_loc("test_attrib/scancode_input/sc-multi-lic.html") with open(expected_file) as exp: expected = exp.read() @@ -304,27 +295,31 @@ def test_scancode_input_multi_lic(self): # expected doesn't work well, it works after removed all the newline and spaces # assert expected == result # assert expected.splitlines(False) == result.splitlines(False) - assert expected.replace('\n', '').replace(' ', '').replace( - '\t', '') == result.replace('\n', '').replace(' ', '').replace('\t', '') + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") def test_generate_with_csv(self): - test_file = get_test_loc( - 'test_attrib/default_template/simple_sample.csv') + test_file = get_test_loc("test_attrib/default_template/simple_sample.csv") errors, abouts = gen.load_inventory(test_file) - lic_dict = {'isc': ['ISC License', - 'isc.LICENSE', - 'Permission to use, copy, modify, and/or distribute this software for any purpose\nwith or without fee is hereby granted, provided that the above copyright notice\nand this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS\nOF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER\nTORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF\nTHIS SOFTWARE.\n', - 'https://scancode-licensedb.aboutcode.org/isc.LICENSE']} + lic_dict = { + "isc": [ + "ISC License", + "isc.LICENSE", + 'Permission to use, copy, modify, and/or distribute this software for any purpose\nwith or without fee is hereby granted, provided that the above copyright notice\nand this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS\nOF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER\nTORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF\nTHIS SOFTWARE.\n', + "https://scancode-licensedb.aboutcode.org/isc.LICENSE", + ] + } is_about_input = False scancode = False error, result = attrib.generate_from_file( - abouts, is_about_input, lic_dict, scancode, min_license_score=0) + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) assert not error - expected_file = get_test_loc( - 'test_attrib/default_template/expect.html') + expected_file = get_test_loc("test_attrib/default_template/expect.html") with open(expected_file) as exp: expected = exp.read() @@ -332,8 +327,9 @@ def test_generate_with_csv(self): result = remove_timestamp(result) expected = remove_timestamp(expected) # assert expected == result - assert expected.replace('\n', '').replace( - ' ', '') == result.replace('\n', '').replace(' ', '') + assert expected.replace("\n", "").replace(" ", "") == result.replace("\n", "").replace( + " ", "" + ) def remove_timestamp(html_text): @@ -341,4 +337,4 @@ def remove_timestamp(html_text): Return the `html_text` generated attribution stripped from timestamps: the timestamp is wrapped in italic block in the default template. """ - return '\n'.join(x for x in html_text.splitlines() if not '' in x) + return "\n".join(x for x in html_text.splitlines() if not "" in x) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index dda13f09..50119e12 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -36,151 +36,149 @@ def test_report_errors(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] - ec = cmd.report_errors(errors, quiet=False, - verbose=True, log_file_loc=None) + ec = cmd.report_errors(errors, quiet=False, verbose=True, log_file_loc=None) assert 6 == ec out, err = capsys.readouterr() expected_out = [ - 'Command completed with 6 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4'] - assert '' == err + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", + ] + assert "" == err assert expected_out == out.splitlines(False) def test_report_errors_without_verbose(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] - ec = cmd.report_errors(errors, quiet=False, - verbose=False, log_file_loc=None) + ec = cmd.report_errors(errors, quiet=False, verbose=False, log_file_loc=None) assert 3 == ec out, err = capsys.readouterr() expected_out = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'WARNING: msg4', + "Command completed with 3 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "WARNING: msg4", ] - assert '' == err + assert "" == err assert expected_out == out.splitlines(False) def test_report_errors_with_quiet_ignores_verbose_flag(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=True) assert severe_errors_count == 6 out, err = capsys.readouterr() - assert '' == out - assert '' == err + assert "" == out + assert "" == err def test_report_errors_with_quiet_ignores_verbose_flag2(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=False) assert severe_errors_count == 3 out, err = capsys.readouterr() - assert '' == out - assert '' == err + assert "" == out + assert "" == err def test_report_errors_with_verbose_flag(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=False, verbose=True) assert severe_errors_count == 6 out, err = capsys.readouterr() expected_out = [ - 'Command completed with 6 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4' + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", ] assert expected_out == out.splitlines(False) - assert '' == err + assert "" == err def test_report_errors_can_write_to_logfile(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] result_file = get_temp_file() - _ec = cmd.report_errors(errors, quiet=False, verbose=True, - log_file_loc=result_file) - with open(result_file, 'r', encoding='utf-8', errors='replace') as rf: + _ec = cmd.report_errors(errors, quiet=False, verbose=True, log_file_loc=result_file) + with open(result_file, "r", encoding="utf-8", errors="replace") as rf: result = rf.read() expected = [ - 'Command completed with 6 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4' + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", ] assert expected == result.splitlines(False) def test_report_errors_does_not_report_duplicate_errors(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), # dupes - Error(WARNING, 'msg4'), - Error(CRITICAL, 'msg1'), + Error(WARNING, "msg4"), + Error(CRITICAL, "msg1"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=True) assert severe_errors_count == 6 @@ -188,85 +186,85 @@ def test_report_errors_does_not_report_duplicate_errors(capsys): def test_get_error_messages(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] emsgs, ec = cmd.get_error_messages(errors) assert 3 == ec expected = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'WARNING: msg4', + "Command completed with 3 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "WARNING: msg4", ] assert expected == emsgs def test_get_error_messages_verbose(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] emsgs, ec = cmd.get_error_messages(errors, verbose=True) assert 6 == ec expected = [ - 'Command completed with 6 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4'] + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", + ] assert expected == emsgs class TestFilterError(unittest.TestCase): - def test_filter_errors_default(self): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] expected = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(WARNING, "msg4"), ] assert expected == cmd.filter_errors(errors) def test_filter_errors_with_min(self): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] expected = [ - Error(CRITICAL, 'msg1'), + Error(CRITICAL, "msg1"), ] assert expected == cmd.filter_errors(errors, CRITICAL) def test_filter_errors_no_errors(self): errors = [ - Error(INFO, 'msg3'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(INFO, "msg3"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] assert [] == cmd.filter_errors(errors) @@ -275,44 +273,36 @@ def test_filter_errors_none(self): class TestParseKeyValues(unittest.TestCase): - def test_parse_key_values_empty(self): assert ({}, []) == cmd.parse_key_values([]) assert ({}, []) == cmd.parse_key_values(None) def test_parse_key_values_simple(self): test = [ - 'key=value', - 'This=THat', - 'keY=bar', + "key=value", + "This=THat", + "keY=bar", ] - expected = { - 'key': 'bar', - 'this': 'THat' - } + expected = {"key": "bar", "this": "THat"} keyvals, errors = cmd.parse_key_values(test) assert expected == keyvals assert not errors def test_parse_key_values_with_errors(self): - test = [ - 'key', - '=THat', - 'keY=', - 'FOO=bar' - ] + test = ["key", "=THat", "keY=", "FOO=bar"] expected = { - 'foo': 'bar', + "foo": "bar", } keyvals, errors = cmd.parse_key_values(test) assert expected == keyvals expected = [ 'missing in "=THat".', 'missing in "keY=".', - 'missing in "key".' + 'missing in "key".', ] assert expected == errors + ############################################################################### # Run full cli command ############################################################################### @@ -327,11 +317,11 @@ def check_about_stdout(options, expected_loc, regen=False): result = run_about_command_test_click(options) if regen: expected_file = get_test_loc(expected_loc, must_exists=False) - with open(expected_file, 'w') as ef: + with open(expected_file, "w") as ef: ef.write(result.output) expected_file = get_test_loc(expected_loc, must_exists=True) - with open(expected_file, 'r') as ef: + with open(expected_file, "r") as ef: expected = ef.read() print(result.output) @@ -339,76 +329,70 @@ def check_about_stdout(options, expected_loc, regen=False): def test_about_help_text(): - check_about_stdout(['--help'], 'test_cmd/help/about_help.txt', regen=False) + check_about_stdout(["--help"], "test_cmd/help/about_help.txt", regen=False) def test_about_inventory_help_text(): check_about_stdout( - ['inventory', '--help'], - 'test_cmd/help/about_inventory_help.txt', regen=False) + ["inventory", "--help"], "test_cmd/help/about_inventory_help.txt", regen=False + ) def test_about_gen_help_text(): - check_about_stdout( - ['gen', '--help'], - 'test_cmd/help/about_gen_help.txt', regen=False) + check_about_stdout(["gen", "--help"], "test_cmd/help/about_gen_help.txt", regen=False) def test_about_gen_license_help_text(): check_about_stdout( - ['gen-license', '--help'], - 'test_cmd/help/about_gen_license_help.txt', regen=False) + ["gen-license", "--help"], "test_cmd/help/about_gen_license_help.txt", regen=False + ) def test_about_check_help_text(): - check_about_stdout( - ['check', '--help'], - 'test_cmd/help/about_check_help.txt', regen=False) + check_about_stdout(["check", "--help"], "test_cmd/help/about_check_help.txt", regen=False) def test_about_attrib_help_text(): - check_about_stdout( - ['attrib', '--help'], - 'test_cmd/help/about_attrib_help.txt', regen=False) + check_about_stdout(["attrib", "--help"], "test_cmd/help/about_attrib_help.txt", regen=False) def test_about_command_fails_with_an_unknown_subcommand(): test_dir = get_temp_dir() - result = run_about_command_test_click(['foo', test_dir], expected_rc=2) - assert 'Error: No such command \'foo\'.' in result.output + result = run_about_command_test_click(["foo", test_dir], expected_rc=2) + assert "Error: No such command 'foo'." in result.output def test_about_inventory_command_can_run_minimally_without_error(): - test_dir = get_test_loc('test_cmd/repository-mini') + test_dir = get_test_loc("test_cmd/repository-mini") result = get_temp_file() - run_about_command_test_click(['inventory', test_dir, result]) + run_about_command_test_click(["inventory", test_dir, result]) def test_about_gen_command_can_run_minimally_without_error(): - test_inv = get_test_loc('test_cmd/geninventory.csv') + test_inv = get_test_loc("test_cmd/geninventory.csv") gen_dir = get_temp_dir() - run_about_command_test_click(['gen', test_inv, gen_dir]) + run_about_command_test_click(["gen", test_inv, gen_dir]) def test_about_attrib_command_can_run_minimally_without_error(): - test_dir = get_test_loc('test_cmd/repository-mini') + test_dir = get_test_loc("test_cmd/repository-mini") result = get_temp_file() - run_about_command_test_click(['attrib', test_dir, result]) + run_about_command_test_click(["attrib", test_dir, result]) def test_about_transform_command_can_run_minimally_without_error(): - test_file = get_test_loc('test_cmd/transform.csv') - result = get_temp_file('file_name.csv') - run_about_command_test_click(['transform', test_file, result]) + test_file = get_test_loc("test_cmd/transform.csv") + result = get_temp_file("file_name.csv") + run_about_command_test_click(["transform", test_file, result]) def test_about_transform_help_text(): check_about_stdout( - ['transform', '--help'], - 'test_cmd/help/about_transform_help.txt', regen=False) + ["transform", "--help"], "test_cmd/help/about_transform_help.txt", regen=False + ) def test_about_transform_expanded_help_text(): check_about_stdout( - ['transform', '--help-format'], - 'test_cmd/help/about_transform_config_help.txt', regen=False) + ["transform", "--help-format"], "test_cmd/help/about_transform_config_help.txt", regen=False + ) diff --git a/tests/test_gen.py b/tests/test_gen.py index 6624c1da..266622d2 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -29,67 +29,82 @@ class GenTest(unittest.TestCase): - def test_check_duplicated_columns(self): - test_file = get_test_loc('test_gen/dup_keys.csv') - expected = [Error( - ERROR, 'Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.')] + test_file = get_test_loc("test_gen/dup_keys.csv") + expected = [ + Error( + ERROR, + "Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.", + ) + ] result = gen.check_duplicated_columns(test_file) assert expected == result def test_check_duplicated_columns_handles_lower_upper_case(self): - test_file = get_test_loc('test_gen/dup_keys_with_diff_case.csv') - expected = [Error( - ERROR, 'Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.')] + test_file = get_test_loc("test_gen/dup_keys_with_diff_case.csv") + expected = [ + Error( + ERROR, + "Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.", + ) + ] result = gen.check_duplicated_columns(test_file) assert expected == result def test_check_duplicated_about_resource(self): - arp_list = ['/test/test.c', 'test/test1.h'] - arp1 = '/test/test.c' - arp2 = '/test/tmp/test.c' - expected = Error(CRITICAL, - "The input has duplicated values in 'about_resource' field: " + arp1) + arp_list = ["/test/test.c", "test/test1.h"] + arp1 = "/test/test.c" + arp2 = "/test/tmp/test.c" + expected = Error( + CRITICAL, "The input has duplicated values in 'about_resource' field: " + arp1 + ) result1 = gen.check_duplicated_about_resource(arp1, arp_list) result2 = gen.check_duplicated_about_resource(arp2, arp_list) assert result1 == expected - assert result2 == '' + assert result2 == "" def test_check_newline_in_file_field(self): - test_dict1 = {'about_resource': '/test/test.c', - 'name': 'test.c', 'notice_file': 'NOTICE\nNOTICE2'} - test_dict2 = {'about_resource': '/test/test.c', - 'name': 'test.c', 'notice_file': 'NOTICE, NOTICE2'} + test_dict1 = { + "about_resource": "/test/test.c", + "name": "test.c", + "notice_file": "NOTICE\nNOTICE2", + } + test_dict2 = { + "about_resource": "/test/test.c", + "name": "test.c", + "notice_file": "NOTICE, NOTICE2", + } expected = [ - Error(CRITICAL, - "New line character detected in 'notice_file' for '/test/test.c' which is not supported." - "\nPlease use ',' to declare multiple files.")] + Error( + CRITICAL, + "New line character detected in 'notice_file' for '/test/test.c' which is not supported." + "\nPlease use ',' to declare multiple files.", + ) + ] result1 = gen.check_newline_in_file_field(test_dict1) result2 = gen.check_newline_in_file_field(test_dict2) assert result1 == expected assert result2 == [] def test_check_about_resource_filename(self): - arp1 = '/test/t@est.c' - arp2 = '/test/t|est.c' - msg = ("Invalid characters present in 'about_resource' " - "field: " + arp2) + arp1 = "/test/t@est.c" + arp2 = "/test/t|est.c" + msg = "Invalid characters present in 'about_resource' field: " + arp2 expected2 = Error(ERROR, msg) result1 = gen.check_about_resource_filename(arp1) result2 = gen.check_about_resource_filename(arp2) - assert result1 == '' + assert result1 == "" assert result2 == expected2 def test_load_inventory(self): - location = get_test_loc('test_gen/inv.csv') + location = get_test_loc("test_gen/inv.csv") base_dir = get_temp_dir() errors, abouts = gen.load_inventory(location, base_dir=base_dir) expected_num_errors = 29 assert len(errors) == expected_num_errors - expected = ( - '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -98,62 +113,57 @@ def test_load_inventory(self): custom1: | multi line -''' - ) +""" result = [a.dumps() for a in abouts] assert expected == result[0] def test_load_inventory_without_about_resource(self): - location = get_test_loc('test_gen/inv_no_about_resource.csv') + location = get_test_loc("test_gen/inv_no_about_resource.csv") base_dir = get_temp_dir() from_attrib = False - errors, abouts = gen.load_inventory( - location, base_dir=base_dir, from_attrib=from_attrib) - expected = ( - '''name: AboutCode + errors, abouts = gen.load_inventory(location, base_dir=base_dir, from_attrib=from_attrib) + expected = """name: AboutCode version: 0.11.0 license_expression: apache-2.0 licenses: - key: apache-2.0 name: apache-2.0 -''' - ) +""" assert errors == [] result = [a.dumps() for a in abouts] assert expected == result[0] def test_load_inventory_without_about_resource_from_attrib(self): - location = get_test_loc('test_gen/inv_no_about_resource.csv') + location = get_test_loc("test_gen/inv_no_about_resource.csv") base_dir = get_temp_dir() from_attrib = True - errors, abouts = gen.load_inventory( - location, base_dir=base_dir, from_attrib=from_attrib) + errors, abouts = gen.load_inventory(location, base_dir=base_dir, from_attrib=from_attrib) expected_num_errors = 0 assert len(errors) == expected_num_errors - expected = ( - '''name: AboutCode + expected = """name: AboutCode version: 0.11.0 license_expression: apache-2.0 licenses: - key: apache-2.0 name: apache-2.0 -''' - ) +""" result = [a.dumps() for a in abouts] assert expected == result[0] def test_load_inventory_with_errors(self): - location = get_test_loc('test_gen/inv4.csv') + location = get_test_loc("test_gen/inv4.csv") base_dir = get_temp_dir() errors, abouts = gen.load_inventory(location, base_dir=base_dir) expected_errors = [ Error( - WARNING, "Field name: ['confirmed copyright'] contains illegal name characters (or empty spaces) and is ignored."), - Error(INFO, 'Field about_resource: Path'), - Error(INFO, "Field ['resource', 'test'] is a custom field.") + WARNING, + "Field name: ['confirmed copyright'] contains illegal name characters (or empty spaces) and is ignored.", + ), + Error(INFO, "Field about_resource: Path"), + Error(INFO, "Field ['resource', 'test'] is a custom field."), ] for exp, err in zip(expected_errors, errors): @@ -161,98 +171,127 @@ def test_load_inventory_with_errors(self): assert err.message.startswith(exp.message) expected = ( - 'about_resource: .\n' - 'name: AboutCode\n' - 'version: 0.11.0\n' - 'description: |\n' - ' multi\n' - ' line\n' + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "description: |\n" + " multi\n" + " line\n" # 'confirmed copyright: Copyright (c) nexB, Inc.\n' - 'resource: this.ABOUT\n' - 'test: This is a test\n' + "resource: this.ABOUT\n" + "test: This is a test\n" ) result = [a.dumps() for a in abouts] assert expected == result[0] def test_load_inventory_simple_xlsx(self): - location = get_test_loc('test_gen/load/simple_sample.xlsx') + location = get_test_loc("test_gen/load/simple_sample.xlsx") base_dir = get_temp_dir() errors, abouts = gen.load_inventory(location, base_dir=base_dir) expected_errors = [] result = [(level, e) for level, e in errors if level > INFO] assert expected_errors == result - assert abouts[0].name.value == 'cryptohash-sha256' - assert abouts[1].name.value == 'some_component' + assert abouts[0].name.value == "cryptohash-sha256" + assert abouts[1].name.value == "some_component" - assert abouts[0].version.value == 'v 0.11.100.1' - assert abouts[1].version.value == 'v 0.0.1' + assert abouts[0].version.value == "v 0.11.100.1" + assert abouts[1].version.value == "v 0.0.1" - assert abouts[0].license_expression.value == 'bsd-new and mit' - assert abouts[1].license_expression.value == 'mit' + assert abouts[0].license_expression.value == "bsd-new and mit" + assert abouts[1].license_expression.value == "mit" def test_load_scancode_json(self): - location = get_test_loc('test_gen/load/clean-text-0.3.0-lceupi.json') + location = get_test_loc("test_gen/load/clean-text-0.3.0-lceupi.json") inventory = gen.load_scancode_json(location) - expected = {'about_resource': 'clean-text-0.3.0', 'type': 'directory', - 'name': 'clean-text-0.3.0', 'base_name': 'clean-text-0.3.0', - 'extension': '', 'size': 0, 'date': None, 'sha1': None, - 'md5': None, 'sha256': None, 'mime_type': None, 'file_type': None, - 'programming_language': None, 'is_binary': False, 'is_text': False, - 'is_archive': False, 'is_media': False, 'is_source': False, - 'is_script': False, 'licenses': [], 'license_expressions': [], - 'percentage_of_license_text': 0, 'copyrights': [], 'holders': [], - 'authors': [], 'packages': [], 'emails': [], 'urls': [], 'files_count': 9, - 'dirs_count': 1, 'size_count': 32826, 'scan_errors': []} + expected = { + "about_resource": "clean-text-0.3.0", + "type": "directory", + "name": "clean-text-0.3.0", + "base_name": "clean-text-0.3.0", + "extension": "", + "size": 0, + "date": None, + "sha1": None, + "md5": None, + "sha256": None, + "mime_type": None, + "file_type": None, + "programming_language": None, + "is_binary": False, + "is_text": False, + "is_archive": False, + "is_media": False, + "is_source": False, + "is_script": False, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 9, + "dirs_count": 1, + "size_count": 32826, + "scan_errors": [], + } # We will only check the first element in the inventory list assert inventory[0] == expected def test_generation_dir_endswith_space(self): - location = get_test_loc( - 'test_gen/inventory/complex/about_file_path_dir_endswith_space.csv') + location = get_test_loc("test_gen/inventory/complex/about_file_path_dir_endswith_space.csv") base_dir = get_temp_dir() errors, _abouts = gen.generate(location, base_dir) - expected_errors_msg1 = 'contains directory name ends with spaces which is not allowed. Generation skipped.' - expected_errors_msg2 = 'Field about_resource' + expected_errors_msg1 = ( + "contains directory name ends with spaces which is not allowed. Generation skipped." + ) + expected_errors_msg2 = "Field about_resource" assert errors assert len(errors) == 2 - assert expected_errors_msg1 in errors[0].message or expected_errors_msg1 in errors[1].message - assert expected_errors_msg2 in errors[0].message or expected_errors_msg2 in errors[1].message + assert ( + expected_errors_msg1 in errors[0].message or expected_errors_msg1 in errors[1].message + ) + assert ( + expected_errors_msg2 in errors[0].message or expected_errors_msg2 in errors[1].message + ) def test_generation_with_no_about_resource(self): - location = get_test_loc('test_gen/inv2.csv') + location = get_test_loc("test_gen/inv2.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = dict([('.', None)]) + expected = dict([(".", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 def test_generation_with_no_about_resource_reference(self): - location = get_test_loc('test_gen/inv3.csv') + location = get_test_loc("test_gen/inv3.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = dict([('test.tar.gz', None)]) + expected = dict([("test.tar.gz", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 - msg = 'Field about_resource' + msg = "Field about_resource" assert msg in errors[0].message def test_generation_with_no_about_resource_reference_no_resource_validation(self): - location = get_test_loc('test_gen/inv3.csv') + location = get_test_loc("test_gen/inv3.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = dict([('test.tar.gz', None)]) + expected = dict([("test.tar.gz", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 def test_generate(self): - location = get_test_loc('test_gen/inv.csv') + location = get_test_loc("test_gen/inv.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) @@ -264,8 +303,7 @@ def test_generate(self): assert msg1 in err_msg_list result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -274,12 +312,11 @@ def test_generate(self): custom1: | multi line -''' - ) +""" assert expected == result def test_generate(self): - location = get_test_loc('test_gen/inv.csv') + location = get_test_loc("test_gen/inv.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) @@ -291,8 +328,7 @@ def test_generate(self): assert msg1 in err_msg_list result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -301,19 +337,17 @@ def test_generate(self): custom1: | multi line -''' - ) +""" assert expected == result def test_generate_multi_lic_issue_443(self): - location = get_test_loc('test_gen/multi_lic_issue_443/test.csv') + location = get_test_loc("test_gen/multi_lic_issue_443/test.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: test + expected = """about_resource: test name: test version: '1.5' licenses: @@ -326,38 +360,33 @@ def test_generate_multi_lic_issue_443(self): - key: License3 name: License3 file: LIC3.LICENSE -''' - ) +""" assert expected == result def test_generate_multi_lic_issue_444(self): - location = get_test_loc('test_gen/multi_lic_issue_444/test1.csv') + location = get_test_loc("test_gen/multi_lic_issue_444/test1.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: test.c + expected = """about_resource: test.c name: test.c licenses: - key: License1 name: License1 file: LIC1.LICENSE, LIC2.LICENSE -''' - ) +""" assert expected == result def test_generate_license_key_with_custom_file_450_no_fetch(self): - location = get_test_loc( - 'test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv') + location = get_test_loc("test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: test.c + expected = """about_resource: test.c name: test.c license_expression: mit AND custom licenses: @@ -366,13 +395,13 @@ def test_generate_license_key_with_custom_file_450_no_fetch(self): - key: custom name: custom - file: custom.txt -''' - ) +""" assert expected == result def test_generate_with_no_license_key_custom_lic_file(self): location = get_test_loc( - 'test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv') + "test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv" + ) base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) @@ -381,18 +410,15 @@ def test_generate_with_no_license_key_custom_lic_file(self): a = abouts[0] result1 = a.dumps() - expected1 = ( - '''about_resource: test.c + expected1 = """about_resource: test.c name: test.c licenses: - file: custom.txt -''' - ) +""" assert expected1 == result1 def test_generate_with_license_key_custom_lic_file(self): - location = get_test_loc( - 'test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv') + location = get_test_loc("test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) @@ -401,44 +427,43 @@ def test_generate_with_license_key_custom_lic_file(self): a = abouts[0] result1 = a.dumps() - expected1 = ( - '''about_resource: test.c + expected1 = """about_resource: test.c name: test.c license_expression: custom licenses: - key: custom name: custom file: custom.txt -''' - ) +""" assert expected1 == result1 def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self): - location = get_test_loc( - 'test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv') + location = get_test_loc("test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - lic_dict = {u'mit': [u'MIT License', - u'mit.LICENSE', - u'This component is released under MIT License.', - u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', - u'mit' - ]} + lic_dict = { + "mit": [ + "MIT License", + "mit.LICENSE", + "This component is released under MIT License.", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "mit", + ] + } # The first row from the test file a = abouts[0] - a.license_key.value.append('mit') - a.license_key.value.append('custom') + a.license_key.value.append("mit") + a.license_key.value.append("custom") result1 = a.dumps(lic_dict) # The second row from the test file b = abouts[1] - b.license_key.value.append('custom') - b.license_key.value.append('mit') + b.license_key.value.append("custom") + b.license_key.value.append("mit") result2 = b.dumps(lic_dict) - expected1 = ( - '''about_resource: test.c + expected1 = """about_resource: test.c name: test.c license_expression: mit AND custom licenses: @@ -450,11 +475,9 @@ def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self): - key: custom name: custom file: custom.txt -''' - ) +""" - expected2 = ( - '''about_resource: test.h + expected2 = """about_resource: test.h name: test.h license_expression: custom AND mit licenses: @@ -466,39 +489,37 @@ def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self): file: mit.LICENSE url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit spdx_license_key: mit -''' - ) +""" assert expected1 == result1 assert expected2 == result2 - @skip('FIXME: this test is making a failed, live API call') + @skip("FIXME: this test is making a failed, live API call") def test_generate_not_overwrite_original_license_file(self): - location = get_test_loc('test_gen/inv5.csv') + location = get_test_loc("test_gen/inv5.csv") base_dir = get_temp_dir() reference_dir = None - fetch_license = ['url', 'lic_key'] + fetch_license = ["url", "lic_key"] - _errors, abouts = gen.generate( - location, base_dir, reference_dir, fetch_license) + _errors, abouts = gen.generate(location, base_dir, reference_dir, fetch_license) - result = [a.dumps()for a in abouts][0] + result = [a.dumps() for a in abouts][0] expected = ( - 'about_resource: .\n' - 'name: AboutCode\n' - 'version: 0.11.0\n' - 'licenses:\n' - ' - file: this.LICENSE\n') + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "licenses:\n" + " - file: this.LICENSE\n" + ) assert expected == result def test_generate_new_lic_fields_563(self): - location = get_test_loc('test_gen/inv7.csv') + location = get_test_loc("test_gen/inv7.csv") base_dir = get_temp_dir() _errors, abouts = gen.generate(location, base_dir) result = [a.dumps() for a in abouts][0] - expected = ( - '''about_resource: test.c + expected = """about_resource: test.c name: test.c license_expression: mit declared_license_expression: isc @@ -507,21 +528,22 @@ def test_generate_new_lic_fields_563(self): licenses: - key: mit name: mit -''' - ) +""" assert expected == result def test_boolean_value_not_lost(self): - location = get_test_loc('test_gen/inv6.csv') + location = get_test_loc("test_gen/inv6.csv") base_dir = get_temp_dir() _errors, abouts = gen.generate(location, base_dir) in_mem_result = [a.dumps() for a in abouts][0] - expected = (u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'redistribute: yes\n' - u'attribute: yes\n' - u'modified: no\n') + expected = ( + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "redistribute: yes\n" + "attribute: yes\n" + "modified: no\n" + ) assert expected == in_mem_result diff --git a/tests/test_model.py b/tests/test_model.py index 48004dbc..b5176276 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -66,8 +66,8 @@ def fix_crlf(items): This is fixing this until we find can why """ for key, value in items: - if isinstance(value, str) and '\r\n' in value: - value = value.replace('\r\n', '\n') + if isinstance(value, str) and "\r\n" in value: + value = value.replace("\r\n", "\n") yield key, value @@ -93,12 +93,11 @@ def get_unicode_content(location): """ Read file at location and return a unicode string. """ - with open(location, encoding='utf-8', errors='replace') as doc: + with open(location, encoding="utf-8", errors="replace") as doc: return doc.read() class FieldTest(unittest.TestCase): - def test_Field_init(self): model.Field() model.StringField() @@ -115,12 +114,12 @@ def test_empty_Field_has_no_content(self): def test_empty_Field_has_default_value(self): field = model.Field() - assert '' == field.value + assert "" == field.value def test_PathField_check_location(self): - test_file = 'license.LICENSE' - field = model.PathField(name='f', value=test_file, present=True) - base_dir = get_test_loc('test_model/base_dir') + test_file = "license.LICENSE" + field = model.PathField(name="f", value=test_file, present=True) + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) expected_errrors = [] @@ -131,68 +130,66 @@ def test_PathField_check_location(self): assert expected == result def test_PathField_check_missing_location(self): - test_file = 'does.not.exist' - field = model.PathField(name='f', value=test_file, present=True) - base_dir = get_test_loc('test_model/base_dir') + test_file = "does.not.exist" + field = model.PathField(name="f", value=test_file, present=True) + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) file_path = posixpath.join(base_dir, test_file) - err_msg = 'Field f: Path %s not found' % file_path + err_msg = "Field f: Path %s not found" % file_path - expected_errors = [ - Error(CRITICAL, err_msg)] + expected_errors = [Error(CRITICAL, err_msg)] assert expected_errors == errors result = field.value[test_file] assert None == result def test_TextField_loads_file(self): - field = model.FileTextField( - name='f', value='license.LICENSE', present=True) + field = model.FileTextField(name="f", value="license.LICENSE", present=True) - base_dir = get_test_loc('test_model/base_dir') + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) assert [] == errors - expected = {'license.LICENSE': 'some license text'} + expected = {"license.LICENSE": "some license text"} assert expected == field.value def test_PackageUrlField_is_valid_url(self): - assert model.PackageUrlField.is_valid_purl('pkg:pypi/saneyaml@0.1') + assert model.PackageUrlField.is_valid_purl("pkg:pypi/saneyaml@0.1") def test_PackageUrlField_is_valid_url_no_version(self): - assert model.PackageUrlField.is_valid_purl('pkg:pypi/saneyaml') + assert model.PackageUrlField.is_valid_purl("pkg:pypi/saneyaml") def test_UrlField_is_valid_url(self): - assert model.UrlField.is_valid_url('http://www.google.com') + assert model.UrlField.is_valid_url("http://www.google.com") def test_UrlField_is_valid_url_not_starting_with_www(self): - assert model.UrlField.is_valid_url('https://nexb.com') - assert model.UrlField.is_valid_url( - 'http://archive.apache.org/dist/httpcomponents/commons-httpclient/2.0/source/commons-httpclient-2.0-alpha2-src.tar.gz') + assert model.UrlField.is_valid_url("https://nexb.com") assert model.UrlField.is_valid_url( - 'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)') - assert model.UrlField.is_valid_url('http://nothing_here.com') + "http://archive.apache.org/dist/httpcomponents/commons-httpclient/2.0/source/commons-httpclient-2.0-alpha2-src.tar.gz" + ) + assert model.UrlField.is_valid_url("http://de.wikipedia.org/wiki/Elf (Begriffsklärung)") + assert model.UrlField.is_valid_url("http://nothing_here.com") def test_UrlField_is_valid_url_no_schemes(self): - assert not model.UrlField.is_valid_url('google.com') - assert not model.UrlField.is_valid_url('www.google.com') - assert not model.UrlField.is_valid_url('') + assert not model.UrlField.is_valid_url("google.com") + assert not model.UrlField.is_valid_url("www.google.com") + assert not model.UrlField.is_valid_url("") def test_UrlField_is_valid_url_not_ends_with_com(self): - assert model.UrlField.is_valid_url('http://www.google') + assert model.UrlField.is_valid_url("http://www.google") def test_UrlField_is_valid_url_ends_with_slash(self): - assert model.UrlField.is_valid_url('http://www.google.co.uk/') + assert model.UrlField.is_valid_url("http://www.google.co.uk/") def test_UrlField_is_valid_url_empty_URL(self): - assert not model.UrlField.is_valid_url('http:') + assert not model.UrlField.is_valid_url("http:") def check_validate(self, field_class, value, expected, expected_errors): """ Check field values after validation """ - field = field_class(name='s', value=value, present=True) + field = field_class(name="s", value=value, present=True) # check that validate can be applied multiple times without side effects for _ in range(2): errors = field.validate() @@ -201,47 +198,47 @@ def check_validate(self, field_class, value, expected, expected_errors): def test_StringField_validate_trailing_spaces_are_removed(self): field_class = model.StringField - value = 'trailin spaces ' - expected = 'trailin spaces' + value = "trailin spaces " + expected = "trailin spaces" self.check_validate(field_class, value, expected, expected_errors=[]) def test_ListField_contains_list_after_validate(self): - value = 'string' + value = "string" field_class = model.ListField expected = [value] self.check_validate(field_class, value, expected, expected_errors=[]) def test_ListField_contains_stripped_strings_after_validate(self): - value = '''first line - second line ''' + value = """first line + second line """ field_class = model.ListField - expected = ['first line', 'second line'] + expected = ["first line", "second line"] self.check_validate(field_class, value, expected, expected_errors=[]) def test_PathField_contains_stripped_strings_after_validate(self): - value = '''first line - second line ''' + value = """first line + second line """ field_class = model.ListField - expected = ['first line', 'second line'] + expected = ["first line", "second line"] self.check_validate(field_class, value, expected, expected_errors=[]) def test_PathField_contains_dict_after_validate(self): - value = 'string' + value = "string" field_class = model.PathField - expected = dict([('string', None)]) + expected = dict([("string", None)]) expected_errors = [ - Error( - ERROR, 'Field s: Unable to verify path: string: No base directory provided') + Error(ERROR, "Field s: Unable to verify path: string: No base directory provided") ] self.check_validate(field_class, value, expected, expected_errors) def test_SingleLineField_has_errors_if_multiline(self): - value = '''line1 - line2''' + value = """line1 + line2""" field_class = model.SingleLineField expected = value expected_errors = [ - Error(ERROR, 'Field s: Cannot span multiple lines: line1\n line2')] + Error(ERROR, "Field s: Cannot span multiple lines: line1\n line2") + ] self.check_validate(field_class, value, expected, expected_errors) @@ -249,151 +246,149 @@ class YamlParseTest(unittest.TestCase): maxDiff = None def test_saneyaml_load_can_parse_simple_fields(self): - test = get_test_content('test_model/parse/basic.about') + test = get_test_content("test_model/parse/basic.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), + ("single_line", "optional"), + ("other_field", "value"), ] assert expected == list(result.items()) def test_saneyaml_load_does_not_convert_to_crlf(self): - test = get_test_content('test_model/crlf/about.ABOUT') + test = get_test_content("test_model/crlf/about.ABOUT") result = saneyaml.load(test) expected = [ - (u'about_resource', u'.'), - (u'name', u'pytest'), - (u'description', u'first line\nsecond line\nthird line\n'), - (u'copyright', u'copyright') + ("about_resource", "."), + ("name", "pytest"), + ("description", "first line\nsecond line\nthird line\n"), + ("copyright", "copyright"), ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_continuations(self): - test = get_test_content('test_model/parse/continuation.about') + test = get_test_content("test_model/parse/continuation.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), - (u'multi_line', u'some value and more and yet more') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more and yet more"), ] assert expected == list(result.items()) def test_saneyaml_load_can_handle_multiline_texts_and_strips_text_fields(self): - test = get_test_content('test_model/parse/complex.about') + test = get_test_content("test_model/parse/complex.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), - ('multi_line', 'some value and more and yet more'), - ('yetanother', 'sdasd')] + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more and yet more"), + ("yetanother", "sdasd"), + ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_verbatim_text_unstripped(self): - test = get_test_content('test_model/parse/continuation_verbatim.about') + test = get_test_content("test_model/parse/continuation_verbatim.about") result = saneyaml.load(test) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'some value \n and more \n and yet more \n \n') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value \n and more \n and yet more \n \n"), ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_verbatim_tab_text_unstripped(self): - test = get_test_content( - 'test_model/parse/continuation_verbatim_with_tab.about') + test = get_test_content("test_model/parse/continuation_verbatim_with_tab.about") data = replace_tab_with_spaces(test) result = saneyaml.load(data) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'This is a long description\nwith tab.\n') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "This is a long description\nwith tab.\n"), ] assert expected == list(result.items()) def test_saneyaml_load_report_error_for_invalid_field_name(self): - test = get_test_content('test_model/parse/invalid_names.about') + test = get_test_content("test_model/parse/invalid_names.about") try: saneyaml.load(test) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass def test_saneyaml_dangling_text_is_not_an_invalid_continuation(self): - test = get_test_content('test_model/parse/invalid_continuation.about') + test = get_test_content("test_model/parse/invalid_continuation.about") result = saneyaml.load(test) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'some value and more\ninvalid continuation2') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more\ninvalid continuation2"), ] assert expected == list(result.items()) def test_saneyaml_load_accepts_unicode_keys_and_values(self): - test = get_test_content( - 'test_model/parse/non_ascii_field_name_value.about') + test = get_test_content("test_model/parse/non_ascii_field_name_value.about") result = saneyaml.load(test) expected = [ - ('name', 'name'), - ('about_resource', '.'), - ('owner', 'Matías Aguirre'), - (u'Matías', u'unicode field name') + ("name", "name"), + ("about_resource", "."), + ("owner", "Matías Aguirre"), + ("Matías", "unicode field name"), ] assert expected == list(result.items()) def test_saneyaml_load_accepts_blank_lines_and_spaces_in_field_names(self): - test = ''' + test = """ name: test space version: 0.7.0 about_resource: about.py field with spaces: This is a test case for field with spaces -''' +""" result = saneyaml.load(test) expected = [ - ('name', 'test space'), - ('version', '0.7.0'), - ('about_resource', 'about.py'), - (u'field with spaces', u'This is a test case for field with spaces'), + ("name", "test space"), + ("version", "0.7.0"), + ("about_resource", "about.py"), + ("field with spaces", "This is a test case for field with spaces"), ] assert expected == list(result.items()) def test_saneyaml_loads_blank_lines_and_lines_without_no_colon(self): - test = ''' + test = """ name: no colon test test version: 0.7.0 about_resource: about.py test with no colon -''' +""" try: saneyaml.load(test) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass class AboutTest(unittest.TestCase): - def test_About_load_ignores_original_field_order_and_uses_standard_predefined_order(self): # fields in this file are not in the standard order - test_file = get_test_loc('test_model/parse/ordered_fields.ABOUT') + test_file = get_test_loc("test_model/parse/ordered_fields.ABOUT") a = model.About(test_file) assert [] == a.errors - expected = ['about_resource', 'name', 'version', 'download_url'] + expected = ["about_resource", "name", "version", "download_url"] result = [f.name for f in a.all_fields() if f.present] assert expected == result @@ -401,13 +396,14 @@ def test_About_duplicate_field_names_are_detected_with_different_case(self): # This test is failing because the YAML does not keep the order when # loads the test files. For instance, it treat the 'About_Resource' as the # first element and therefore the dup key is 'about_resource'. - test_file = get_test_loc('test_model/parse/dupe_field_name.ABOUT') + test_file = get_test_loc("test_model/parse/dupe_field_name.ABOUT") a = model.About(test_file) expected = [ Error( - WARNING, 'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"'), - Error( - WARNING, 'Field Name is a duplicate. Original value: "old" replaced with: "new"') + WARNING, + 'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"', + ), + Error(WARNING, 'Field Name is a duplicate. Original value: "old" replaced with: "new"'), ] result = a.errors @@ -417,31 +413,33 @@ def test_About_duplicate_field_names_are_not_reported_if_same_value(self): # This test is failing because the YAML does not keep the order when # loads the test files. For instance, it treat the 'About_Resource' as the # first element and therefore the dup key is 'about_resource'. - test_file = get_test_loc( - 'test_model/parse/dupe_field_name_no_new_value.ABOUT') + test_file = get_test_loc("test_model/parse/dupe_field_name_no_new_value.ABOUT") a = model.About(test_file) - expected = [ - ] + expected = [] result = a.errors assert sorted(expected) == sorted(result) def check_About_hydrate(self, about, fields): - expected = set([ - 'name', - 'homepage_url', - 'download_url', - 'version', - 'copyright', - 'date', - 'license_spdx', - 'license_text_file', - 'notice_file', - 'about_resource']) + expected = set( + [ + "name", + "homepage_url", + "download_url", + "version", + "copyright", + "date", + "license_spdx", + "license_text_file", + "notice_file", + "about_resource", + ] + ) expected_errors = [ - Error(INFO, 'Custom Field: date'), - Error(INFO, 'Custom Field: license_spdx'), - Error(INFO, 'Custom Field: license_text_file')] + Error(INFO, "Custom Field: date"), + Error(INFO, "Custom Field: license_spdx"), + Error(INFO, "Custom Field: license_text_file"), + ] errors = about.hydrate(fields) @@ -451,180 +449,166 @@ def check_About_hydrate(self, about, fields): assert expected == result def test_About_hydrate_normalize_field_names_to_lowercase(self): - test_content = get_test_content( - 'test_gen/parser_tests/upper_field_names.ABOUT') + test_content = get_test_content("test_gen/parser_tests/upper_field_names.ABOUT") fields = saneyaml.load(test_content).items() a = model.About() for _ in range(3): self.check_About_hydrate(a, fields) def test_About_with_existing_about_resource_has_no_error(self): - test_file = get_test_loc( - 'test_gen/parser_tests/about_resource_field.ABOUT') + test_file = get_test_loc("test_gen/parser_tests/about_resource_field.ABOUT") a = model.About(test_file) assert [] == a.errors - result = a.about_resource.value['about_resource.c'] + result = a.about_resource.value["about_resource.c"] # this means we have a location self.assertNotEqual([], result) def test_About_loads_ignored_resources_field(self): # fields in this file are not in the standard order - test_file = get_test_loc( - 'test_model/parse/with_ignored_resources.ABOUT') + test_file = get_test_loc("test_model/parse/with_ignored_resources.ABOUT") a = model.About(test_file) # assert [] == a.errors - expected = ['about_resource', 'ignored_resources', 'name'] + expected = ["about_resource", "ignored_resources", "name"] result = [f.name for f in a.all_fields() if f.present] assert expected == result def test_About_has_no_errors_when_about_resource_is_missing(self): - test_file = get_test_loc('test_gen/parser_tests/.ABOUT') + test_file = get_test_loc("test_gen/parser_tests/.ABOUT") a = model.About(test_file) assert a.errors == [] def test_About_has_errors_when_about_resource_does_not_exist(self): - test_file = get_test_loc( - 'test_gen/parser_tests/missing_about_ref.ABOUT') - file_path = posixpath.join(posixpath.dirname( - test_file), 'about_file_missing.c') + test_file = get_test_loc("test_gen/parser_tests/missing_about_ref.ABOUT") + file_path = posixpath.join(posixpath.dirname(test_file), "about_file_missing.c") a = model.About(test_file) - err_msg = 'Field about_resource: Path %s not found' % file_path + err_msg = "Field about_resource: Path %s not found" % file_path expected = [Error(INFO, err_msg)] result = a.errors assert expected == result def test_About_has_errors_when_missing_required_fields_are_missing(self): - test_file = get_test_loc('test_model/parse/missing_required.ABOUT') + test_file = get_test_loc("test_model/parse/missing_required.ABOUT") a = model.About(test_file) expected = [ - Error(CRITICAL, 'Field name is required'), + Error(CRITICAL, "Field name is required"), ] result = a.errors assert expected == result def test_About_has_errors_when_required_fields_are_empty(self): - test_file = get_test_loc('test_model/parse/empty_required.ABOUT') + test_file = get_test_loc("test_model/parse/empty_required.ABOUT") a = model.About(test_file) expected = [ - Error(CRITICAL, 'Field name is required and empty'), + Error(CRITICAL, "Field name is required and empty"), ] result = a.errors assert expected == result def test_About_has_errors_with_empty_notice_file_field(self): - test_file = get_test_loc('test_model/parse/empty_notice_field.about') + test_file = get_test_loc("test_model/parse/empty_notice_field.about") a = model.About(test_file) - expected = [ - Error(INFO, 'Field notice_file is present but empty.')] + expected = [Error(INFO, "Field notice_file is present but empty.")] result = a.errors assert expected == result def test_About_custom_fields_are_never_ignored(self): - test_file = get_test_loc( - 'test_model/custom_fields/custom_fields.about') + test_file = get_test_loc("test_model/custom_fields/custom_fields.about") a = model.About(test_file) result = [(n, f.value) for n, f in a.custom_fields.items()] expected = [ - (u'single_line', u'README STUFF'), - (u'multi_line', u'line1\nline2'), - (u'other', u'sasasas'), - (u'empty', u'') + ("single_line", "README STUFF"), + ("multi_line", "line1\nline2"), + ("other", "sasasas"), + ("empty", ""), ] assert expected == result def test_About_custom_fields_are_not_ignored_and_order_is_preserved(self): - test_file = get_test_loc( - 'test_model/custom_fields/custom_fields.about') + test_file = get_test_loc("test_model/custom_fields/custom_fields.about") a = model.About(test_file) result = [(n, f.value) for n, f in a.custom_fields.items()] expected = [ - (u'single_line', u'README STUFF'), - (u'multi_line', u'line1\nline2'), - (u'other', u'sasasas'), - (u'empty', u'') + ("single_line", "README STUFF"), + ("multi_line", "line1\nline2"), + ("other", "sasasas"), + ("empty", ""), ] assert sorted(expected) == sorted(result) def test_About_has_errors_for_illegal_custom_field_name(self): - test_file = get_test_loc('test_model/parse/illegal_custom_field.about') + test_file = get_test_loc("test_model/parse/illegal_custom_field.about") a = model.About(test_file) expected_errors = [ - Error(INFO, 'Custom Field: hydrate'), - Error( - CRITICAL, "Internal error with custom field: 'hydrate': 'illegal name'.") + Error(INFO, "Custom Field: hydrate"), + Error(CRITICAL, "Internal error with custom field: 'hydrate': 'illegal name'."), ] assert expected_errors == a.errors - assert not hasattr(getattr(a, 'hydrate'), 'value') + assert not hasattr(getattr(a, "hydrate"), "value") field = list(a.custom_fields.values())[0] - assert 'hydrate' == field.name - assert 'illegal name' == field.value + assert "hydrate" == field.name + assert "illegal name" == field.value def test_About_file_fields_are_empty_if_present_and_path_missing(self): - test_file = get_test_loc( - 'test_model/parse/missing_notice_license_files.ABOUT') + test_file = get_test_loc("test_model/parse/missing_notice_license_files.ABOUT") a = model.About(test_file) - file_path1 = posixpath.join( - posixpath.dirname(test_file), 'test.LICENSE') - file_path2 = posixpath.join( - posixpath.dirname(test_file), 'test.NOTICE') + file_path1 = posixpath.join(posixpath.dirname(test_file), "test.LICENSE") + file_path2 = posixpath.join(posixpath.dirname(test_file), "test.NOTICE") - err_msg1 = Error( - CRITICAL, 'Field license_file: Path %s not found' % file_path1) - err_msg2 = Error( - CRITICAL, 'Field notice_file: Path %s not found' % file_path2) + err_msg1 = Error(CRITICAL, "Field license_file: Path %s not found" % file_path1) + err_msg2 = Error(CRITICAL, "Field notice_file: Path %s not found" % file_path2) expected_errors = [err_msg1, err_msg2] assert expected_errors == a.errors - assert {'test.LICENSE': None} == a.license_file.value - assert {'test.NOTICE': None} == a.notice_file.value + assert {"test.LICENSE": None} == a.license_file.value + assert {"test.NOTICE": None} == a.notice_file.value def test_About_notice_and_license_text_are_loaded_from_file(self): - test_file = get_test_loc( - 'test_model/parse/license_file_notice_file.ABOUT') + test_file = get_test_loc("test_model/parse/license_file_notice_file.ABOUT") a = model.About(test_file) - expected = '''Tester holds the copyright for test component. Tester relinquishes copyright of + expected = """Tester holds the copyright for test component. Tester relinquishes copyright of this software and releases the component to Public Domain. -* Email Test@tester.com for any questions''' +* Email Test@tester.com for any questions""" - result = a.license_file.value['license_text.LICENSE'] + result = a.license_file.value["license_text.LICENSE"] assert expected == result - expected = '''Test component is released to Public Domain.''' - result = a.notice_file.value['notice_text.NOTICE'] + expected = """Test component is released to Public Domain.""" + result = a.notice_file.value["notice_text.NOTICE"] assert expected == result def test_About_license_and_notice_text_are_empty_if_field_missing(self): - test_file = get_test_loc('test_model/parse/no_file_fields.ABOUT') + test_file = get_test_loc("test_model/parse/no_file_fields.ABOUT") a = model.About(test_file) assert [] == a.errors assert {} == a.license_file.value assert {} == a.notice_file.value def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self): - test_file = get_test_loc( - 'test_model/parse/non_ascii_field_name_value.about') + test_file = get_test_loc("test_model/parse/non_ascii_field_name_value.about") a = model.About(test_file) expected = [ Error( - WARNING, "Field name: ['mat\xedas'] contains illegal name characters (or empty spaces) and is ignored.") + WARNING, + "Field name: ['mat\xedas'] contains illegal name characters (or empty spaces) and is ignored.", + ) ] assert expected == a.errors def test_About_invalid_boolean_value(self): - test_file = get_test_loc('test_model/parse/invalid_boolean.about') + test_file = get_test_loc("test_model/parse/invalid_boolean.about") a = model.About(test_file) expected_msg = "Field modified: Invalid flag value: 'blah'" assert expected_msg in a.errors[0].message def test_About_boolean_value(self): - test_file = get_test_loc('test_model/parse/boolean_data.about') + test_file = get_test_loc("test_model/parse/boolean_data.about") a = model.About(test_file) expected_msg = "Field track_changes is present but empty." assert expected_msg in a.errors[0].message @@ -645,7 +629,7 @@ def test_About_boolean_value(self): assert a.track_changes.value is None def test_About_boolean_numberic_value(self): - test_file = get_test_loc('test_model/parse/boolean_numeric_data.about') + test_file = get_test_loc("test_model/parse/boolean_numeric_data.about") a = model.About(test_file) expected_msg = "Field track_changes is present but empty." assert expected_msg in a.errors[0].message @@ -659,14 +643,14 @@ def test_About_boolean_numberic_value(self): redistribute: yes track_changes: """ - assert a.attribute.value == '3' + assert a.attribute.value == "3" assert a.modified.value is True assert a.internal_use_only.value is False assert a.redistribute.value is True assert a.track_changes.value is None def test_About_boolean_character_value(self): - test_file = get_test_loc('test_model/parse/boolean_chara_data.about') + test_file = get_test_loc("test_model/parse/boolean_chara_data.about") a = model.About(test_file) # Context of the test file """ @@ -674,12 +658,11 @@ def test_About_boolean_character_value(self): name: data attribute: 11 """ - assert a.attribute.value == '11' + assert a.attribute.value == "11" assert len(a.errors) == 0 def test_About_boolean_more_than_2_character_value(self): - test_file = get_test_loc( - 'test_model/parse/boolean_more_than_2_chara_data.about') + test_file = get_test_loc("test_model/parse/boolean_more_than_2_chara_data.about") a = model.About(test_file) expected_msg = "Path: None - Field attribute: Invalid value: 'abc' is not one of: ('yes', 'y', 'true', 'x', 'no', 'n', 'false') and it is not a 1 or 2 character value." assert expected_msg in a.errors[0].message @@ -692,91 +675,94 @@ def test_About_boolean_more_than_2_character_value(self): assert a.attribute.value is None def test_About_contains_about_file_path(self): - test_file = get_test_loc('test_model/serialize/about.ABOUT') + test_file = get_test_loc("test_model/serialize/about.ABOUT") # TODO: I am not sure this override of the about_file_path makes sense - a = model.About(test_file, about_file_path='complete/about.ABOUT') + a = model.About(test_file, about_file_path="complete/about.ABOUT") assert [] == a.errors - expected = 'complete/about.ABOUT' + expected = "complete/about.ABOUT" result = a.about_file_path assert expected == result def test_About_equals(self): - test_file = get_test_loc('test_model/equal/complete/about.ABOUT') - a = model.About(test_file, about_file_path='complete/about.ABOUT') - b = model.About(test_file, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/equal/complete/about.ABOUT") + a = model.About(test_file, about_file_path="complete/about.ABOUT") + b = model.About(test_file, about_file_path="complete/about.ABOUT") assert a == b def test_About_are_not_equal_with_small_text_differences(self): - test_file = get_test_loc('test_model/equal/complete2/about.ABOUT') - a = model.About(test_file, about_file_path='complete2/about.ABOUT') - test_file2 = get_test_loc('test_model/equal/complete/about.ABOUT') - b = model.About(test_file2, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/equal/complete2/about.ABOUT") + a = model.About(test_file, about_file_path="complete2/about.ABOUT") + test_file2 = get_test_loc("test_model/equal/complete/about.ABOUT") + b = model.About(test_file2, about_file_path="complete/about.ABOUT") assert a.dumps() != b.dumps() assert a != b def test_get_field_names_only_returns_non_empties(self): a = model.About() - a.custom_fields['f'] = model.StringField( - name='f', value='1', present=True) + a.custom_fields["f"] = model.StringField(name="f", value="1", present=True) b = model.About() - b.custom_fields['g'] = model.StringField( - name='g', value='1', present=True) + b.custom_fields["g"] = model.StringField(name="g", value="1", present=True) abouts = [a, b] # ensure all fields (including custom fields) and # about_resource are collected in the correct order - expected = ['name', 'f', 'g'] + expected = ["name", "f", "g"] result = model.get_field_names(abouts) assert expected == result def test_get_field_names_does_not_return_duplicates_custom_fields(self): a = model.About() - a.custom_fields['f'] = model.StringField(name='f', value='1', - present=True) - a.custom_fields['cf'] = model.StringField(name='cf', value='1', - present=True) + a.custom_fields["f"] = model.StringField(name="f", value="1", present=True) + a.custom_fields["cf"] = model.StringField(name="cf", value="1", present=True) b = model.About() - b.custom_fields['g'] = model.StringField(name='g', value='1', - present=True) - b.custom_fields['cf'] = model.StringField(name='cf', value='2', - present=True) + b.custom_fields["g"] = model.StringField(name="g", value="1", present=True) + b.custom_fields["cf"] = model.StringField(name="cf", value="2", present=True) abouts = [a, b] # ensure all fields (including custom fields) and # about_resource are collected in the correct order expected = [ - 'name', - 'cf', - 'f', - 'g', + "name", + "cf", + "f", + "g", ] result = model.get_field_names(abouts) assert expected == result def test_comma_in_license(self): - test_file = get_test_loc('test_model/special_char/about.ABOUT') + test_file = get_test_loc("test_model/special_char/about.ABOUT") a = model.About(test_file) - expected = Error( - ERROR, "The following character(s) cannot be in the license_key: [',']") + expected = Error(ERROR, "The following character(s) cannot be in the license_key: [',']") assert a.errors[0] == expected def test_load_dict_issue_433(self): package_data = { - 'about_resource': 'package1.zip', - 'name': 'package', - 'version': '1.0', - 'copyright': 'copyright on package', - 'license_expression': 'license1 AND license2', - 'notice_file': 'package1.zip.NOTICE', - 'licenses': [ - {'key': 'license1', 'name': 'License1', 'file': 'license1.LICENSE', - 'url': 'some_url', 'spdx_license_key': 'key'}, - {'key': 'license2', 'name': 'License2', 'file': 'license2.LICENSE', - 'url': 'some_url', 'spdx_license_key': 'key'}, + "about_resource": "package1.zip", + "name": "package", + "version": "1.0", + "copyright": "copyright on package", + "license_expression": "license1 AND license2", + "notice_file": "package1.zip.NOTICE", + "licenses": [ + { + "key": "license1", + "name": "License1", + "file": "license1.LICENSE", + "url": "some_url", + "spdx_license_key": "key", + }, + { + "key": "license2", + "name": "License2", + "file": "license2.LICENSE", + "url": "some_url", + "spdx_license_key": "key", + }, ], } about = model.About() - about.load_dict(package_data, base_dir='') + about.load_dict(package_data, base_dir="") as_dict = about.as_dict() - expected = '''about_resource: package1.zip + expected = """about_resource: package1.zip name: package version: '1.0' license_expression: license1 AND license2 @@ -793,20 +779,21 @@ def test_load_dict_issue_433(self): file: license2.LICENSE url: some_url spdx_license_key: key -''' - lic_dict = {u'license1': [u'License1', u'license1.LICENSE', u'', u'some_url', 'key'], u'license2': [ - u'License2', u'license2.LICENSE', u'', u'some_url', 'key']} +""" + lic_dict = { + "license1": ["License1", "license1.LICENSE", "", "some_url", "key"], + "license2": ["License2", "license2.LICENSE", "", "some_url", "key"], + } assert about.dumps(lic_dict) == expected class SerializationTest(unittest.TestCase): - def test_About_dumps(self): - test_file = get_test_loc('test_model/dumps/about.ABOUT') + test_file = get_test_loc("test_model/dumps/about.ABOUT") a = model.About(test_file) assert [] == a.errors - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -825,38 +812,38 @@ def test_About_dumps(self): - key: apache-2.0 name: Apache 2.0 file: apache-2.0.LICENSE -''' +""" result = a.dumps() assert expected == result def test_About_dumps_does_all_non_empty_present_fields(self): - test_file = get_test_loc('test_model/parse/complete2/about.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about.ABOUT") a = model.About(test_file) expected_error = [ - Error(INFO, 'Custom Field: custom1'), - Error(INFO, 'Custom Field: custom2'), - Error(INFO, 'Field custom2 is present but empty.') + Error(INFO, "Custom Field: custom1"), + Error(INFO, "Custom Field: custom2"), + Error(INFO, "Field custom2 is present but empty."), ] assert sorted(expected_error) == sorted(a.errors) - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 custom1: | multi line -''' +""" result = a.dumps() assert expected == result def test_About_dumps_with_different_boolean_value(self): - test_file = get_test_loc('test_model/parse/complete2/about2.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about2.ABOUT") a = model.About(test_file) expected_error_msg = "Field track_changes: Invalid flag value: 'blah' is not one of" assert len(a.errors) == 1 assert expected_error_msg in a.errors[0].message - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 @@ -864,46 +851,46 @@ def test_About_dumps_with_different_boolean_value(self): redistribute: no attribute: yes modified: yes -''' +""" result = a.dumps() assert set(expected) == set(result) def test_About_dumps_all_non_empty_fields(self): - test_file = get_test_loc('test_model/parse/complete2/about.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about.ABOUT") a = model.About(test_file) expected_error = [ - Error(INFO, 'Custom Field: custom1'), - Error(INFO, 'Custom Field: custom2'), - Error(INFO, 'Field custom2 is present but empty.') + Error(INFO, "Custom Field: custom1"), + Error(INFO, "Custom Field: custom2"), + Error(INFO, "Field custom2 is present but empty."), ] assert sorted(expected_error) == sorted(a.errors) - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 custom1: | multi line -''' +""" result = a.dumps() assert expected == result def test_About_as_dict_contains_special_paths(self): - test_file = get_test_loc('test_model/special/about.ABOUT') - a = model.About(test_file, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/special/about.ABOUT") + a = model.About(test_file, about_file_path="complete/about.ABOUT") expected_errors = [] assert expected_errors == a.errors as_dict = a.as_dict() - expected = 'complete/about.ABOUT' + expected = "complete/about.ABOUT" result = as_dict[model.About.ABOUT_FILE_PATH_ATTR] assert expected == result def test_load_dump_is_idempotent(self): - test_file = get_test_loc('test_model/this.ABOUT') + test_file = get_test_loc("test_model/this.ABOUT") a = model.About() a.load(test_file) - dumped_file = get_temp_file('that.ABOUT') + dumped_file = get_temp_file("that.ABOUT") a.dump(dumped_file) expected = get_unicode_content(test_file).splitlines() @@ -911,60 +898,62 @@ def test_load_dump_is_idempotent(self): # Ignore comment and empty line filtered_result = [] for line in result: - if not line.startswith('#') and not line == '': + if not line.startswith("#") and not line == "": filtered_result.append(line) assert expected == filtered_result def test_load_can_load_unicode(self): - test_file = get_test_loc('test_model/unicode/nose-selecttests.ABOUT') + test_file = get_test_loc("test_model/unicode/nose-selecttests.ABOUT") a = model.About() a.load(test_file) - file_path = posixpath.join(posixpath.dirname( - test_file), 'nose-selecttests-0.3.zip') - err_msg = 'Field about_resource: Path %s not found' % file_path + file_path = posixpath.join(posixpath.dirname(test_file), "nose-selecttests-0.3.zip") + err_msg = "Field about_resource: Path %s not found" % file_path errors = [ - Error(INFO, 'Custom Field: dje_license'), - Error(INFO, 'Custom Field: license_text_file'), - Error(INFO, 'Custom Field: scm_tool'), - Error(INFO, 'Custom Field: scm_repository'), - Error(INFO, 'Custom Field: test'), - Error(INFO, err_msg)] + Error(INFO, "Custom Field: dje_license"), + Error(INFO, "Custom Field: license_text_file"), + Error(INFO, "Custom Field: scm_tool"), + Error(INFO, "Custom Field: scm_repository"), + Error(INFO, "Custom Field: test"), + Error(INFO, err_msg), + ] assert errors == a.errors - assert 'Copyright (c) 2012, Domen Kožar' == a.copyright.value + assert "Copyright (c) 2012, Domen Kožar" == a.copyright.value def test_load_non_unicode(self): - test_file = get_test_loc('test_model/unicode/not-unicode.ABOUT') + test_file = get_test_loc("test_model/unicode/not-unicode.ABOUT") a = model.About() a.load(test_file) err = a.errors[0] assert CRITICAL == err.severity - assert 'Cannot load invalid ABOUT file' in err.message + assert "Cannot load invalid ABOUT file" in err.message def test_as_dict_load_dict_ignores_empties(self): test = { - 'about_resource': '.', - 'author': '', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'custom_empty': '', - 'description': 'AboutCode is a tool\nfor files.', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.'} + "about_resource": ".", + "author": "", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "custom_empty": "", + "description": "AboutCode is a tool\nfor files.", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + } expected = { - 'about_file_path': None, - 'about_resource': dict([('.', None)]), - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'description': 'AboutCode is a tool\nfor files.', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.'} + "about_file_path": None, + "about_resource": dict([(".", None)]), + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "description": "AboutCode is a tool\nfor files.", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + } a = model.About() - base_dir = 'some_dir' + base_dir = "some_dir" a.load_dict(test, base_dir) as_dict = a.as_dict() # FIXME: why converting back to dict? @@ -972,224 +961,227 @@ def test_as_dict_load_dict_ignores_empties(self): def test_load_dict_as_dict_is_idempotent_ignoring_special(self): test = { - 'about_resource': ['.'], - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0'} + "about_resource": ["."], + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", + } a = model.About() - base_dir = 'some_dir' + base_dir = "some_dir" a.load_dict(test, base_dir) as_dict = a.as_dict() expected = { - 'about_file_path': None, - 'about_resource': dict([('.', None)]), - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0'} + "about_file_path": None, + "about_resource": dict([(".", None)]), + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", + } assert expected == dict(as_dict) def test_about_model_class_from_dict_constructor(self): about_data = { - 'about_resource': ['.'], - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0', + "about_resource": ["."], + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", } about = model.About.from_dict(about_data) assert isinstance(about, model.About) - about_data.update({ - 'about_file_path': None, - 'about_resource': dict([('.', None)]), - }) + about_data.update( + { + "about_file_path": None, + "about_resource": dict([(".", None)]), + } + ) assert about_data == about.as_dict() def test_write_output_csv(self): - path = 'test_model/this.ABOUT' + path = "test_model/this.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='csv') + model.write_output([abouts], result, format="csv") - expected = get_test_loc('test_model/expected.csv') + expected = get_test_loc("test_model/expected.csv") check_csv(expected, result) def test_write_output_csv_with_multiple_files(self): - path = 'test_model/multiple_files.ABOUT' + path = "test_model/multiple_files.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='csv') + model.write_output([abouts], result, format="csv") - expected = get_test_loc('test_model/multiple_files_expected.csv') + expected = get_test_loc("test_model/multiple_files_expected.csv") check_csv(expected, result) def test_write_output_json(self): - path = 'test_model/this.ABOUT' + path = "test_model/this.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='json') + model.write_output([abouts], result, format="json") - expected = get_test_loc('test_model/expected.json') + expected = get_test_loc("test_model/expected.json") check_json(expected, result) def test_android_module_license(self): - path = 'test_model/android/single_license.c.ABOUT' + path = "test_model/android/single_license.c.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) parent_dir = get_temp_dir() abouts.android_module_license(parent_dir) - assert os.path.exists(os.path.join( - parent_dir, 'MODULE_LICENSE_PUBLIC_DOMAIN')) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_PUBLIC_DOMAIN")) def test_android_module_multi_licenses(self): - path = 'test_model/android/multi_license.c.ABOUT' + path = "test_model/android/multi_license.c.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) parent_dir = get_temp_dir() abouts.android_module_license(parent_dir) - assert os.path.exists(os.path.join( - parent_dir, 'MODULE_LICENSE_BSD_NEW')) - assert os.path.exists(os.path.join( - parent_dir, 'MODULE_LICENSE_BSD_SIMPLIFIED')) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_BSD_NEW")) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_BSD_SIMPLIFIED")) def test_android_notice(self): - path = 'test_model/android/single_license.c.ABOUT' + path = "test_model/android/single_license.c.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) parent_dir = get_temp_dir() notice_path, notice_context = abouts.android_notice(parent_dir) - expected_path = os.path.join(parent_dir, 'NOTICE') + expected_path = os.path.join(parent_dir, "NOTICE") assert os.path.normpath(notice_path) == expected_path - expected_notice = '''Copyright (c) xyz + expected_notice = """Copyright (c) xyz This component is released to the public domain by the author. -''' +""" assert notice_context == expected_notice class CollectorTest(unittest.TestCase): - def test_collect_inventory_return_errors(self): - test_loc = get_test_loc('test_model/collect_inventory_errors') + test_loc = get_test_loc("test_model/collect_inventory_errors") errors, _abouts = model.collect_inventory(test_loc) - file_path1 = posixpath.join(test_loc, 'distribute_setup.py') - file_path2 = posixpath.join(test_loc, 'date_test.py') + file_path1 = posixpath.join(test_loc, "distribute_setup.py") + file_path2 = posixpath.join(test_loc, "date_test.py") - err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1 - err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2 + err_msg1 = ( + "non-supported_date_format.ABOUT: Field about_resource: Path %s not found" % file_path1 + ) + err_msg2 = ( + "supported_date_format.ABOUT: Field about_resource: Path %s not found" % file_path2 + ) expected_errors = [ Error(INFO, "Field ['date'] is a custom field."), Error(INFO, err_msg1), - Error(INFO, err_msg2)] + Error(INFO, err_msg2), + ] assert sorted(expected_errors) == sorted(errors) def test_collect_inventory_with_long_path(self): - test_loc = extract_test_loc('test_model/longpath.zip') + test_loc = extract_test_loc("test_model/longpath.zip") _errors, abouts = model.collect_inventory(test_loc) assert 2 == len(abouts) expected_paths = ( - 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/non-supported_date_format.ABOUT', - 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/supported_date_format.ABOUT' + "longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/non-supported_date_format.ABOUT", + "longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/supported_date_format.ABOUT", ) results = [a.about_file_path for a in abouts] assert all(r.endswith(expected_paths) for r in results) - expected_name = ['distribute', 'date_test'] + expected_name = ["distribute", "date_test"] result_name = [a.name.value for a in abouts] assert sorted(expected_name) == sorted(result_name) def test_collect_inventory_can_collect_a_single_file(self): - test_loc = get_test_loc( - 'test_model/single_file/django_snippets_2413.ABOUT') + test_loc = get_test_loc("test_model/single_file/django_snippets_2413.ABOUT") _errors, abouts = model.collect_inventory(test_loc) assert 1 == len(abouts) - expected = ['single_file/django_snippets_2413.ABOUT'] + expected = ["single_file/django_snippets_2413.ABOUT"] result = [a.about_file_path for a in abouts] assert expected == result def test_collect_inventory_return_no_warnings_and_model_can_use_relative_paths(self): - test_loc = get_test_loc('test_model/rel/allAboutInOneDir') + test_loc = get_test_loc("test_model/rel/allAboutInOneDir") errors, _abouts = model.collect_inventory(test_loc) expected_errors = [] result = [(level, e) for level, e in errors if level > INFO] assert expected_errors == result def test_collect_inventory_populate_about_file_path(self): - test_loc = get_test_loc('test_model/inventory/complete') + test_loc = get_test_loc("test_model/inventory/complete") errors, abouts = model.collect_inventory(test_loc) assert [] == errors - expected = 'about.ABOUT' + expected = "about.ABOUT" result = abouts[0].about_file_path assert expected == result def test_collect_inventory_with_multi_line(self): - test_loc = get_test_loc( - 'test_model/parse/multi_line_license_expresion.ABOUT') + test_loc = get_test_loc("test_model/parse/multi_line_license_expresion.ABOUT") errors, abouts = model.collect_inventory(test_loc) assert [] == errors expected_lic_url = [ - 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', - 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0'] + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0", + ] returned_lic_url = abouts[0].license_url.value assert expected_lic_url == returned_lic_url def test_collect_inventory_with_license_expression(self): - test_loc = get_test_loc( - 'test_model/parse/multi_line_license_expresion.ABOUT') + test_loc = get_test_loc("test_model/parse/multi_line_license_expresion.ABOUT") errors, abouts = model.collect_inventory(test_loc) assert [] == errors - expected_lic = 'mit or apache-2.0' + expected_lic = "mit or apache-2.0" returned_lic = abouts[0].license_expression.value assert expected_lic == returned_lic def test_collect_inventory_always_collects_custom_fieldsg(self): - test_loc = get_test_loc('test_model/inventory/custom_fields.ABOUT') + test_loc = get_test_loc("test_model/inventory/custom_fields.ABOUT") errors, abouts = model.collect_inventory(test_loc) expected_msg = "Field ['resource', 'custom_mapping'] is a custom field." assert len(errors) == 1 @@ -1198,30 +1190,28 @@ def test_collect_inventory_always_collects_custom_fieldsg(self): assert abouts[0].resource.value def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields(self): - test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT') + test_loc = get_test_loc("test_model/inventory/custom_fields2.ABOUT") errors, abouts = model.collect_inventory(test_loc) - expected_errors = [ - Error( - INFO, "Field ['resource', 'custom_mapping'] is a custom field.") - ] + expected_errors = [Error(INFO, "Field ['resource', 'custom_mapping'] is a custom field.")] assert expected_errors == errors - expected = [ - u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n'] + expected = ["about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n"] assert expected == [a.dumps() for a in abouts] def test_parse_license_expression(self): spec_char, returned_lic, _invalid_lic_exp = model.parse_license_expression( - 'mit or apache-2.0') - expected_lic = ['mit', 'apache-2.0'] + "mit or apache-2.0" + ) + expected_lic = ["mit", "apache-2.0"] expected_spec_char = [] assert expected_lic == returned_lic assert expected_spec_char == spec_char def test_parse_license_expression_with_special_chara(self): spec_char, returned_lic, _invalid_lic_exp = model.parse_license_expression( - 'mit, apache-2.0') + "mit, apache-2.0" + ) expected_lic = [] - expected_spec_char = [','] + expected_spec_char = [","] assert expected_lic == returned_lic assert expected_spec_char == spec_char @@ -1229,91 +1219,91 @@ def test_collect_inventory_works_with_relative_paths(self): # FIXME: This test need to be run under src/attributecode/ # or otherwise it will fail as the test depends on the launching # location - test_loc = get_test_loc('test_model/inventory/relative') + test_loc = get_test_loc("test_model/inventory/relative") # Use '.' as the indication of the current directory - test_loc1 = test_loc + '/./' + test_loc1 = test_loc + "/./" # Use '..' to go back to the parent directory - test_loc2 = test_loc + '/../relative' + test_loc2 = test_loc + "/../relative" errors1, abouts1 = model.collect_inventory(test_loc1) errors2, abouts2 = model.collect_inventory(test_loc2) assert [] == errors1 assert [] == errors2 - expected = 'about.ABOUT' + expected = "about.ABOUT" result1 = abouts1[0].about_file_path result2 = abouts2[0].about_file_path assert expected == result1 assert expected == result2 def test_collect_inventory_basic_from_directory(self): - location = get_test_loc('test_model/inventory/basic') + location = get_test_loc("test_model/inventory/basic") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") expected_errors = [] assert expected_errors == errors - expected = get_test_loc('test_model/inventory/basic/expected.csv') + expected = get_test_loc("test_model/inventory/basic/expected.csv") check_csv(expected, result) def test_collect_inventory_with_about_resource_path_from_directory(self): - location = get_test_loc( - 'test_model/inventory/basic_with_about_resource_path') + location = get_test_loc("test_model/inventory/basic_with_about_resource_path") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") expected_errors = [] assert expected_errors == errors - expected = get_test_loc( - 'test_model/inventory/basic_with_about_resource_path/expected.csv') + expected = get_test_loc("test_model/inventory/basic_with_about_resource_path/expected.csv") check_csv(expected, result) def test_collect_inventory_with_no_about_resource_from_directory(self): - location = get_test_loc('test_model/inventory/no_about_resource_key') + location = get_test_loc("test_model/inventory/no_about_resource_key") errors, abouts = model.collect_inventory(location) assert errors == [] def test_collect_inventory_complex_from_directory(self): - location = get_test_loc('test_model/inventory/complex') + location = get_test_loc("test_model/inventory/complex") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") assert all(e.severity == INFO for e in errors) - expected = get_test_loc('test_model/inventory/complex/expected.csv') + expected = get_test_loc("test_model/inventory/complex/expected.csv") check_csv(expected, result, fix_cell_linesep=True, regen=False) def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(self): - location = get_test_loc('test_model/crlf/about.ABOUT') + location = get_test_loc("test_model/crlf/about.ABOUT") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") assert all(e.severity == INFO for e in errors) - expected = get_test_loc('test_model/crlf/expected.csv') + expected = get_test_loc("test_model/crlf/expected.csv") check_csv(expected, result, fix_cell_linesep=True, regen=False) def test_copy_redist_src_no_structure(self): - test_loc = get_test_loc('test_model/redistribution/') - copy_list = [get_test_loc('test_model/redistribution/this.c'), - get_test_loc('test_model/redistribution/test/subdir')] + test_loc = get_test_loc("test_model/redistribution/") + copy_list = [ + get_test_loc("test_model/redistribution/this.c"), + get_test_loc("test_model/redistribution/test/subdir"), + ] output = get_temp_dir() - expected_file = ['this.c', 'subdir'] + expected_file = ["this.c", "subdir"] with_structure = False - err = model.copy_redist_src( - copy_list, test_loc, output, with_structure) + err = model.copy_redist_src(copy_list, test_loc, output, with_structure) assert err == [] from os import listdir + copied_files = listdir(output) assert len(expected_file) == len(copied_files) assert err == [] @@ -1321,20 +1311,22 @@ def test_copy_redist_src_no_structure(self): assert file in copied_files def test_copy_redist_src_with_structure(self): - test_loc = get_test_loc('test_model/redistribution/') - copy_list = [get_test_loc('test_model/redistribution/this.c'), - get_test_loc('test_model/redistribution/test/subdir')] + test_loc = get_test_loc("test_model/redistribution/") + copy_list = [ + get_test_loc("test_model/redistribution/this.c"), + get_test_loc("test_model/redistribution/test/subdir"), + ] output = get_temp_dir() - expected_file = ['this.c', 'test'] + expected_file = ["this.c", "test"] with_structure = True - err = model.copy_redist_src( - copy_list, test_loc, output, with_structure) + err = model.copy_redist_src(copy_list, test_loc, output, with_structure) assert err == [] from os import listdir + copied_files = listdir(output) assert len(expected_file) == len(copied_files) assert err == [] @@ -1342,13 +1334,12 @@ def test_copy_redist_src_with_structure(self): assert file in copied_files def test_get_copy_list(self): - location = get_test_loc('test_model/redistribution/') + location = get_test_loc("test_model/redistribution/") result = get_temp_file() errors, abouts = model.collect_inventory(location) copy_list, err = model.get_copy_list(abouts, location) assert err == [] - expected = [os.path.join(location, 'this.c'), - os.path.join(location, 'test/subdir')] + expected = [os.path.join(location, "this.c"), os.path.join(location, "test/subdir")] if on_windows: norm_list = [] for c in copy_list: @@ -1359,20 +1350,19 @@ def test_get_copy_list(self): class FetchLicenseTest(unittest.TestCase): - - @mock.patch.object(model, 'get') + @mock.patch.object(model, "get") def test_valid_api_url(self, mock_data): - mock_data.return_value = '' - assert model.valid_api_url('non_valid_url') is False + mock_data.return_value = "" + assert model.valid_api_url("non_valid_url") is False - @mock.patch('attributecode.util.have_network_connection') - @mock.patch('attributecode.model.valid_api_url') + @mock.patch("attributecode.util.have_network_connection") + @mock.patch("attributecode.model.valid_api_url") def test_pre_process_and_fetch_license_dict_dje(self, have_network_connection, valid_api_url): have_network_connection.return_value = True valid_api_url.return_value = False error_msg = ( - 'Network problem. Please check your Internet connection. ' - 'License generation is skipped.') + "Network problem. Please check your Internet connection. License generation is skipped." + ) expected = ({}, [Error(ERROR, error_msg)]) assert model.pre_process_and_fetch_license_dict([]) == expected @@ -1380,14 +1370,16 @@ def test_pre_process_and_fetch_license_dict_dje(self, have_network_connection, v expected = ({}, []) assert model.pre_process_and_fetch_license_dict([]) == expected - @mock.patch('attributecode.util.have_network_connection') - @mock.patch('attributecode.model.valid_api_url') - def test_pre_process_and_fetch_license_dict_licensedb(self, have_network_connection, valid_api_url): + @mock.patch("attributecode.util.have_network_connection") + @mock.patch("attributecode.model.valid_api_url") + def test_pre_process_and_fetch_license_dict_licensedb( + self, have_network_connection, valid_api_url + ): have_network_connection.return_value = False valid_api_url.return_value = False error_msg = ( - 'Network problem. Please check your Internet connection. ' - 'License generation is skipped.') + "Network problem. Please check your Internet connection. License generation is skipped." + ) expected = ({}, [Error(ERROR, error_msg)]) assert model.pre_process_and_fetch_license_dict([]) == expected diff --git a/tests/test_transform.py b/tests/test_transform.py index b3f583e6..ab3c3a65 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -30,230 +30,502 @@ class TransformTest(unittest.TestCase): - def test_transform_data_new_col(self): - data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), (u'Component', u'test.c'), - (u'version', '1'), (u'notes', u'test'), (u'temp', u'foo')])] - configuration = get_test_loc('test_transform/configuration_new_cols') + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration_new_cols") transformer = Transformer.from_file(configuration) data, err = transform_data(data, transformer) - expected_data = [dict(OrderedDict([(u'path', u'/tmp/test.c'), - (u'about_resource', u'/tmp/test.c'), - (u'name', u'test.c'), (u'version', u'1'), - (u'notes', u'test'), (u'temp', u'foo')]))] + expected_data = [ + dict( + OrderedDict( + [ + ("path", "/tmp/test.c"), + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ) + ] assert len(data) == len(expected_data) for d in data: assert dict(d) in expected_data def test_transform_data(self): - data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), - (u'Component', u'test.c'), (u'version', u'1'), - (u'notes', u'test'), (u'temp', u'foo')])] - configuration = get_test_loc('test_transform/configuration') + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") transformer = Transformer.from_file(configuration) data, err = transform_data(data, transformer) - expect_name = [u'about_resource', u'name', u'version'] - expected_data = [dict(OrderedDict( - [(u'about_resource', u'/tmp/test.c'), (u'name', u'test.c'), (u'version', u'1')]))] + expect_name = ["about_resource", "name", "version"] + expected_data = [ + dict( + OrderedDict( + [("about_resource", "/tmp/test.c"), ("name", "test.c"), ("version", "1")] + ) + ) + ] assert len(data) == len(expected_data) for d in data: assert dict(d) in expected_data def test_transform_data_mutli_rows(self): - data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), (u'Component', u'test.c'), (u'Confirmed Version', u'v0.01')]), - OrderedDict([(u'Directory/Filename', u'/tmp/tmp.h'), (u'Component', u'tmp.h'), (u'Confirmed Version', None)])] - configuration = get_test_loc('test_transform/configuration2') + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("Confirmed Version", "v0.01"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/tmp/tmp.h"), + ("Component", "tmp.h"), + ("Confirmed Version", None), + ] + ), + ] + configuration = get_test_loc("test_transform/configuration2") transformer = Transformer.from_file(configuration) data, err = transform_data(data, transformer) - expect_name = [u'about_resource', u'name', u'version'] - expected_data = [dict(OrderedDict([(u'about_resource', u'/tmp/test.c'), (u'name', u'test.c'), (u'version', u'v0.01')])), - dict(OrderedDict([(u'about_resource', u'/tmp/tmp.h'), (u'name', u'tmp.h'), (u'version', None)]))] + expect_name = ["about_resource", "name", "version"] + expected_data = [ + dict( + OrderedDict( + [("about_resource", "/tmp/test.c"), ("name", "test.c"), ("version", "v0.01")] + ) + ), + dict( + OrderedDict( + [("about_resource", "/tmp/tmp.h"), ("name", "tmp.h"), ("version", None)] + ) + ), + ] assert len(data) == len(expected_data) for d in data: assert dict(d) in expected_data def test_normalize_dict_data_scancode(self): - test_file = get_test_loc('test_transform/input_scancode.json') + test_file = get_test_loc("test_transform/input_scancode.json") json_data = read_json(test_file) data = normalize_dict_data(json_data) - expected_data = [OrderedDict([(u'path', u'samples'), - (u'type', u'directory'), - (u'name', u'samples'), - (u'base_name', u'samples'), - (u'extension', u''), (u'size', 0), - (u'date', None), (u'sha1', - None), (u'md5', None), - (u'mime_type', None), (u'file_type', None), - (u'programming_language', None), - (u'is_binary', False), (u'is_text', False), - (u'is_archive', False), (u'is_media', False), - (u'is_source', False), (u'is_script', False), - (u'licenses', []), (u'license_expressions', []), - (u'copyrights', []), (u'holders', []), - (u'authors', []), (u'packages', []), - (u'emails', []), (u'urls', []), - (u'files_count', 33), (u'dirs_count', 10), - (u'size_count', 1161083), (u'scan_errors', [])])] + expected_data = [ + OrderedDict( + [ + ("path", "samples"), + ("type", "directory"), + ("name", "samples"), + ("base_name", "samples"), + ("extension", ""), + ("size", 0), + ("date", None), + ("sha1", None), + ("md5", None), + ("mime_type", None), + ("file_type", None), + ("programming_language", None), + ("is_binary", False), + ("is_text", False), + ("is_archive", False), + ("is_media", False), + ("is_source", False), + ("is_script", False), + ("licenses", []), + ("license_expressions", []), + ("copyrights", []), + ("holders", []), + ("authors", []), + ("packages", []), + ("emails", []), + ("urls", []), + ("files_count", 33), + ("dirs_count", 10), + ("size_count", 1161083), + ("scan_errors", []), + ] + ) + ] assert data == expected_data def test_normalize_dict_data_json(self): - json_data = OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'), - (u'Component', u'AboutCode-toolkit'), - (u'version', u'1.2.3'), (u'note', u'test'), - (u'temp', u'foo')]) + json_data = OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.2.3"), + ("note", "test"), + ("temp", "foo"), + ] + ) data = normalize_dict_data(json_data) - expected_data = [OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'), - (u'Component', u'AboutCode-toolkit'), - (u'version', u'1.2.3'), (u'note', u'test'), - (u'temp', u'foo')])] + expected_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.2.3"), + ("note", "test"), + ("temp", "foo"), + ] + ) + ] assert data == expected_data def test_normalize_dict_data_json_array(self): - json_data = [OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'), - (u'Component', u'AboutCode-toolkit'), - (u'version', u'1.0'), (u'temp', u'fpp')]), - OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'), - (u'Component', u'AboutCode-toolkit1'), - (u'version', u'1.1'), (u'temp', u'foo')])] + json_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.0"), + ("temp", "fpp"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit1/"), + ("Component", "AboutCode-toolkit1"), + ("version", "1.1"), + ("temp", "foo"), + ] + ), + ] data = normalize_dict_data(json_data) - expected_data = [OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit/'), - (u'Component', u'AboutCode-toolkit'), - (u'version', u'1.0'), (u'temp', u'fpp')]), - OrderedDict([(u'Directory/Filename', u'/aboutcode-toolkit1/'), - (u'Component', u'AboutCode-toolkit1'), - (u'version', u'1.1'), - (u'temp', u'foo')])] + expected_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.0"), + ("temp", "fpp"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit1/"), + ("Component", "AboutCode-toolkit1"), + ("version", "1.1"), + ("temp", "foo"), + ] + ), + ] assert data == expected_data def test_check_duplicate_fields(self): - field_name = ['path', 'name', 'path', 'version'] - expected = ['path'] + field_name = ["path", "name", "path", "version"] + expected = ["path"] dups = check_duplicate_fields(field_name) assert dups == expected def test_strip_trailing_fields_csv(self): - test = [u'about_resource', u'name ', u' version '] - expected = [u'about_resource', u'name', u'version'] + test = ["about_resource", "name ", " version "] + expected = ["about_resource", "name", "version"] result = strip_trailing_fields_csv(test) assert result == expected def test_strip_trailing_fields_json(self): - test = [OrderedDict([(u'about_resource', u'/this.c'), - (u'name ', u'this.c'), (u' version ', u'0.11.0')])] - expected = [OrderedDict( - [(u'about_resource', u'/this.c'), (u'name', u'this.c'), (u'version', u'0.11.0')])] + test = [ + OrderedDict( + [("about_resource", "/this.c"), ("name ", "this.c"), (" version ", "0.11.0")] + ) + ] + expected = [ + OrderedDict([("about_resource", "/this.c"), ("name", "this.c"), ("version", "0.11.0")]) + ] result = strip_trailing_fields_json(test) assert result == expected def test_read_excel(self): - test_file = get_test_loc('test_transform/simple.xlsx') + test_file = get_test_loc("test_transform/simple.xlsx") error, data = read_excel(test_file) assert not error - expected = [OrderedDict([('about_resource', '/test.c'), ('name', 'test.c'), ('license_expression', 'mit')]), - OrderedDict([('about_resource', '/test2.c'), ('name', 'test2.c'), ('license_expression', 'mit and apache-2.0')])] + expected = [ + OrderedDict( + [("about_resource", "/test.c"), ("name", "test.c"), ("license_expression", "mit")] + ), + OrderedDict( + [ + ("about_resource", "/test2.c"), + ("name", "test2.c"), + ("license_expression", "mit and apache-2.0"), + ] + ), + ] assert data == expected def test_read_csv_rows(self): - test_file = get_test_loc('test_transform/simple.csv') + test_file = get_test_loc("test_transform/simple.csv") data = read_csv_rows(test_file) - expected = [['about_resource', 'name', 'license_expression'], - ['/test.c', 'test.c', 'mit'], - ['/test2.c', 'test2.c', 'mit and apache-2.0']] + expected = [ + ["about_resource", "name", "license_expression"], + ["/test.c", "test.c", "mit"], + ["/test2.c", "test2.c", "mit and apache-2.0"], + ] assert list(data) == expected def test_transform_csv(self): - test_file = get_test_loc('test_transform/input.csv') + test_file = get_test_loc("test_transform/input.csv") data, err = transform_csv(test_file) - expected = [{'Directory/Filename': '/aboutcode-toolkit/', - 'Component': 'AboutCode-toolkit', - 'Confirmed Version': '123', 'notes': ''}] + expected = [ + { + "Directory/Filename": "/aboutcode-toolkit/", + "Component": "AboutCode-toolkit", + "Confirmed Version": "123", + "notes": "", + } + ] assert len(err) == 0 assert data == expected def test_transform_excel(self): - test_file = get_test_loc('test_transform/input.xlsx') + test_file = get_test_loc("test_transform/input.xlsx") data, err = transform_excel(test_file) - expected = [OrderedDict([('Directory/Filename', '/aboutcode-toolkit/'), - ('Component', 'AboutCode-toolkit'), - ('Confirmed Version', 123), ('notes', '')])] + expected = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("Confirmed Version", 123), + ("notes", ""), + ] + ) + ] assert len(err) == 0 assert data == expected def test_transform_json(self): - test_file = get_test_loc('test_transform/input.json') + test_file = get_test_loc("test_transform/input.json") data, err = transform_json(test_file) - expected = [{'Directory/Filename': '/aboutcode-toolkit/', - 'Component': 'AboutCode-toolkit', - 'Confirmed Version': '123', 'notes': ''}] + expected = [ + { + "Directory/Filename": "/aboutcode-toolkit/", + "Component": "AboutCode-toolkit", + "Confirmed Version": "123", + "notes": "", + } + ] assert len(err) == 0 assert data == expected def test_apply_renamings(self): - data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), - (u'Component', u'test.c'), (u'version', u'1'), - (u'notes', u'test'), (u'temp', u'foo')])] - configuration = get_test_loc('test_transform/configuration') + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") transformer = Transformer.from_file(configuration) - expected = [OrderedDict([(u'about_resource', u'/tmp/test.c'), (u'name', - u'test.c'), (u'version', u'1'), (u'notes', u'test'), (u'temp', u'foo')])] + expected = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] renamed_field_data = transformer.apply_renamings(data) assert renamed_field_data == expected def test_apply_renamings_nested_list(self): - data = [{'path': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [ - {'score': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}] - configuration = get_test_loc('test_transform/configuration3') + data = [ + { + "path": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] + configuration = get_test_loc("test_transform/configuration3") transformer = Transformer.from_file(configuration) - expected = [{'about_resource': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [ - {'score_renamed': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score_renamed': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}] + expected = [ + { + "about_resource": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score_renamed": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score_renamed": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] updated_data = transformer.apply_renamings(data) assert updated_data == expected def test_filter_excluded(self): - data = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), - (u'Component', u'test.c'), (u'version', u'1'), - (u'notes', u'test'), (u'temp', u'foo')])] - configuration = get_test_loc('test_transform/configuration') + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") transformer = Transformer.from_file(configuration) - expected = [OrderedDict([(u'Directory/Filename', u'/tmp/test.c'), (u'Component', - u'test.c'), (u'version', u'1'), (u'notes', u'test')])] + expected = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ] + ) + ] updated_data = transformer.filter_excluded(data) assert updated_data == expected def test_filter_excluded_nested_list(self): - data = [{'path': 'samples/JGroups-error.log', 'type': 'file', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [ - {'score': 90.0, 'start_line': 4, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'start_line': 5, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}] - configuration = get_test_loc('test_transform/configuration3') + data = [ + { + "path": "samples/JGroups-error.log", + "type": "file", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] + configuration = get_test_loc("test_transform/configuration3") transformer = Transformer.from_file(configuration) - expected = [{'path': 'samples/JGroups-error.log', 'name': 'JGroups-error.log', 'license_detections': [{'license_expression': 'apache-1.1 AND apache-2.0', 'matches': [ - {'score': 90.0, 'end_line': 4, 'license_expression': 'apache-1.1'}, {'score': 100.0, 'end_line': 5, 'license_expression': 'apache-2.0'}]}]}] + expected = [ + { + "path": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + {"score": 90.0, "end_line": 4, "license_expression": "apache-1.1"}, + {"score": 100.0, "end_line": 5, "license_expression": "apache-2.0"}, + ], + } + ], + } + ] updated_data = transformer.filter_excluded(data) assert updated_data == expected def test_filter_fields(self): - data = [OrderedDict([(u'about_resource', u'/tmp/test.c'), - (u'name', u'test.c'), (u'version', u'1'), - (u'notes', u'test'), (u'temp', u'foo')])] - configuration = get_test_loc('test_transform/configuration') + data = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") transformer = Transformer.from_file(configuration) updated_data = transformer.filter_fields(data) - expected = [OrderedDict([(u'about_resource', u'/tmp/test.c'), - (u'name', u'test.c'), (u'version', u'1'), - (u'temp', u'foo')])] + expected = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("temp", "foo"), + ] + ) + ] for d in updated_data: assert dict(d) in expected diff --git a/tests/test_util.py b/tests/test_util.py index 45007141..3bf14045 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -32,154 +32,154 @@ class TestResourcePaths(unittest.TestCase): - def test_resource_name(self): - expected = 'first' - result = util.resource_name('some/things/first') + expected = "first" + result = util.resource_name("some/things/first") assert expected == result def test_resource_name_with_extension(self): - expected = 'first.ABOUT' - result = util.resource_name('/some/things/first.ABOUT') + expected = "first.ABOUT" + result = util.resource_name("/some/things/first.ABOUT") assert expected == result def test_resource_name_for_dir(self): - expected = 'first' - result = util.resource_name('some/things/first/') + expected = "first" + result = util.resource_name("some/things/first/") assert expected == result def test_resource_name_windows(self): - expected = r'first.' - result = util.resource_name(r'c:\some\things\first.') + expected = r"first." + result = util.resource_name(r"c:\some\things\first.") assert expected == result def test_resource_name_mixed_windows_posix(self): - expected = r'first' - result = util.resource_name(r'c:\some/things\first') + expected = r"first" + result = util.resource_name(r"c:\some/things\first") assert expected == result def test_resource_name_double_slash(self): - expected = 'first' - result = util.resource_name(r'some\thi ngs//first') + expected = "first" + result = util.resource_name(r"some\thi ngs//first") assert expected == result def test_resource_name_punctuation(self): - expected = '_$asafg:' - result = util.resource_name('%6571351()2/75612$/_$asafg:') + expected = "_$asafg:" + result = util.resource_name("%6571351()2/75612$/_$asafg:") assert expected == result def test_resource_name_simple_slash(self): - expected = '' - result = util.resource_name('/') + expected = "" + result = util.resource_name("/") assert expected == result def test_resource_name_spaces(self): - expected = '' - result = util.resource_name('/ / ') + expected = "" + result = util.resource_name("/ / ") assert expected == result def test_resource_name_does_not_recurse_infinitely(self): - expected = '' - result = util.resource_name(' / ') + expected = "" + result = util.resource_name(" / ") assert expected == result def test_to_posix_from_win(self): - test = r'c:\this\that' - expected = 'c:/this/that' + test = r"c:\this\that" + expected = "c:/this/that" result = util.to_posix(test) assert expected == result def test_to_posix_from_posix(self): - test = r'/this/that' - expected = '/this/that' + test = r"/this/that" + expected = "/this/that" result = util.to_posix(test) assert expected == result def test_to_posix_from_mixed(self): - test = r'/this/that\this' - expected = '/this/that/this' + test = r"/this/that\this" + expected = "/this/that/this" result = util.to_posix(test) assert expected == result def test_to_native_from_win(self): - test = r'c:\this\that' + test = r"c:\this\that" if on_posix: - expected = 'c:/this/that' + expected = "c:/this/that" else: expected = test result = util.to_native(test) assert expected == result def test_to_native_from_posix(self): - test = r'/this/that' + test = r"/this/that" if on_windows: - expected = r'\this\that' + expected = r"\this\that" else: expected = test result = util.to_native(test) assert expected == result def test_to_native_from_mixed(self): - test = r'/this/that\this' + test = r"/this/that\this" if on_windows: - expected = r'\this\that\this' + expected = r"\this\that\this" else: - expected = r'/this/that/this' + expected = r"/this/that/this" result = util.to_native(test) assert expected == result def test_invalid_chars_with_valid_chars(self): - name = string.digits + string.ascii_letters + '_-.+()~[]{}@%!$,' + name = string.digits + string.ascii_letters + "_-.+()~[]{}@%!$," result = util.invalid_chars(name) expected = [] assert expected == result def test_space_is_valid_chars(self): - result = util.invalid_chars(' ') + result = util.invalid_chars(" ") expected = [] assert expected == result def test_invalid_chars_with_invalid_in_name_and_dir(self): - result = util.invalid_chars('_$as/afg:') - expected = [':'] + result = util.invalid_chars("_$as/afg:") + expected = [":"] assert expected == result def test_invalid_chars_in_file_name(self): - name = '%657!1351()275612$_$asafg:~|[]{}+-.' + name = "%657!1351()275612$_$asafg:~|[]{}+-." result = util.invalid_chars(name) - expected = [':', '|'] + expected = [":", "|"] assert expected == result def test_invalid_chars_with_space_is_valid(self): - result = util.invalid_chars('_ Hello') + result = util.invalid_chars("_ Hello") expected = [] assert expected == result def test_check_file_names_with_dupes_return_errors(self): - paths = ['some/path', 'some/PAth'] + paths = ["some/path", "some/PAth"] result = util.check_file_names(paths) expected = [ Error( CRITICAL, - "Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name") + "Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name", + ) ] assert expected == result def test_check_file_names_without_dupes_return_no_error(self): - paths = ['some/path', - 'some/otherpath'] + paths = ["some/path", "some/otherpath"] result = util.check_file_names(paths) expected = [] assert expected == result def test_check_file_names_with_no_invalid_char_return_no_error(self): paths = [ - 'locations/file', - 'locations/file1', - 'locations/file2', - 'locations/dir1/file2', - 'locations/dir1/dir2/file1', - 'locations/dir2/file1'] + "locations/file", + "locations/file1", + "locations/file2", + "locations/dir1/file2", + "locations/dir1/dir2/file1", + "locations/dir2/file1", + ] expected = [] result = util.check_file_names(paths) @@ -187,123 +187,143 @@ def test_check_file_names_with_no_invalid_char_return_no_error(self): def test_check_file_names_with_invalid_chars_return_errors(self): paths = [ - 'locations/file', - 'locations/file with space', - 'locations/dir1/dir2/file1', - 'locations/dir2/file1', - 'Accessibilité/ périmètre', - 'locations/in:valid' + "locations/file", + "locations/file with space", + "locations/dir1/dir2/file1", + "locations/dir2/file1", + "Accessibilité/ périmètre", + "locations/in:valid", ] import sys + if sys.version_info[0] < 3: # python2 - expected = [Error( - CRITICAL, b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'")] + expected = [ + Error( + CRITICAL, + b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'", + ) + ] else: expected = [ - Error(CRITICAL, "Invalid characters ':' in file name at: 'locations/in:valid'")] + Error(CRITICAL, "Invalid characters ':' in file name at: 'locations/in:valid'") + ] result = util.check_file_names(paths) assert expected[0].message == result[0].message assert expected == result def test_is_about_file(self): - assert util.is_about_file('test.About') - assert util.is_about_file('test2.aboUT') - assert not util.is_about_file('no_about_ext.something') - assert not util.is_about_file('about') - assert not util.is_about_file('about.txt') + assert util.is_about_file("test.About") + assert util.is_about_file("test2.aboUT") + assert not util.is_about_file("no_about_ext.something") + assert not util.is_about_file("about") + assert not util.is_about_file("about.txt") def test_is_about_file_is_false_if_only_bare_extension(self): - assert not util.is_about_file('.ABOUT') + assert not util.is_about_file(".ABOUT") def test_get_relative_path(self): - test = [('/some/path', '/some/path/file', 'file'), - ('path', '/path/file', 'file'), - ('/path', '/path/file', 'file'), - ('/path/', '/path/file/', 'file'), - ('/path/', 'path/', 'path'), - ('/p1/p2/p3', '/p1/p2//p3/file', 'file'), - (r'c:\some/path', 'c:/some/path/file', 'file'), - (r'c:\\some\\path\\', 'c:/some/path/file', 'file'), - ] + test = [ + ("/some/path", "/some/path/file", "file"), + ("path", "/path/file", "file"), + ("/path", "/path/file", "file"), + ("/path/", "/path/file/", "file"), + ("/path/", "path/", "path"), + ("/p1/p2/p3", "/p1/p2//p3/file", "file"), + (r"c:\some/path", "c:/some/path/file", "file"), + (r"c:\\some\\path\\", "c:/some/path/file", "file"), + ] for base_loc, full_loc, expected in test: result = util.get_relative_path(base_loc, full_loc) assert expected == result def test_get_relative_path_with_same_path_twice(self): - test = [('/some/path/file', 'path/file'), - ('/path/file', 'path/file'), - ('/path/file/', 'path/file'), - ('path/', 'path'), - ('/p1/p2//p3/file', 'p3/file'), - ('c:/some/path/file', 'path/file'), - (r'c:\\some\\path\\file', 'path/file'), - ] + test = [ + ("/some/path/file", "path/file"), + ("/path/file", "path/file"), + ("/path/file/", "path/file"), + ("path/", "path"), + ("/p1/p2//p3/file", "p3/file"), + ("c:/some/path/file", "path/file"), + (r"c:\\some\\path\\file", "path/file"), + ] for loc, expected in test: result = util.get_relative_path(loc, loc) assert expected == result class TestGetLocations(unittest.TestCase): - def test_get_locations(self): - test_dir = get_test_loc('test_util/about_locations') - expected = sorted([ - 'file with_spaces.ABOUT', - 'file1', - 'file2', - 'dir1/file2', - 'dir1/file2.aBout', - 'dir1/dir2/file1.about', - 'dir2/file1']) + test_dir = get_test_loc("test_util/about_locations") + expected = sorted( + [ + "file with_spaces.ABOUT", + "file1", + "file2", + "dir1/file2", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + "dir2/file1", + ] + ) result = sorted(util.get_locations(test_dir)) - result = [l.partition('/about_locations/')[-1] for l in result] + result = [l.partition("/about_locations/")[-1] for l in result] assert expected == result def test_get_about_locations(self): - test_dir = get_test_loc('test_util/about_locations') - expected = sorted([ - 'file with_spaces.ABOUT', - 'dir1/file2.aBout', - 'dir1/dir2/file1.about', - ]) + test_dir = get_test_loc("test_util/about_locations") + expected = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + ] + ) result = sorted(util.get_about_locations(test_dir)) - result = [l.partition('/about_locations/')[-1] for l in result] + result = [l.partition("/about_locations/")[-1] for l in result] assert expected == result def test_get_about_locations_with_exclude(self): - test_dir = get_test_loc('test_util/about_locations') - exclude1 = ('dir*',) - exclude2 = ('*dir2*',) - exclude3 = ('*test*',) - exclude4 = ('dir1/',) - expected1 = sorted([ - 'file with_spaces.ABOUT', - ]) - expected2 = sorted([ - 'file with_spaces.ABOUT', - 'dir1/file2.aBout', - ]) - expected3 = sorted([ - 'file with_spaces.ABOUT', - 'dir1/file2.aBout', - 'dir1/dir2/file1.about', - ]) - expected4 = sorted([ - 'file with_spaces.ABOUT', - ]) + test_dir = get_test_loc("test_util/about_locations") + exclude1 = ("dir*",) + exclude2 = ("*dir2*",) + exclude3 = ("*test*",) + exclude4 = ("dir1/",) + expected1 = sorted( + [ + "file with_spaces.ABOUT", + ] + ) + expected2 = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + ] + ) + expected3 = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + ] + ) + expected4 = sorted( + [ + "file with_spaces.ABOUT", + ] + ) result1 = sorted(util.get_about_locations(test_dir, exclude1)) result2 = sorted(util.get_about_locations(test_dir, exclude2)) result3 = sorted(util.get_about_locations(test_dir, exclude3)) result4 = sorted(util.get_about_locations(test_dir, exclude4)) - result1 = [l.partition('/about_locations/')[-1] for l in result1] - result2 = [l.partition('/about_locations/')[-1] for l in result2] - result3 = [l.partition('/about_locations/')[-1] for l in result3] - result4 = [l.partition('/about_locations/')[-1] for l in result4] + result1 = [l.partition("/about_locations/")[-1] for l in result1] + result2 = [l.partition("/about_locations/")[-1] for l in result2] + result3 = [l.partition("/about_locations/")[-1] for l in result3] + result4 = [l.partition("/about_locations/")[-1] for l in result4] assert expected1 == result1 assert expected2 == result2 @@ -311,223 +331,265 @@ def test_get_about_locations_with_exclude(self): assert expected4 == result4 def test_get_locations_can_yield_a_single_file(self): - test_file = get_test_loc( - 'test_util/about_locations/file with_spaces.ABOUT') + test_file = get_test_loc("test_util/about_locations/file with_spaces.ABOUT") result = list(util.get_locations(test_file)) assert 1 == len(result) def test_get_about_locations_for_about(self): - location = get_test_loc('test_util/get_about_locations') + location = get_test_loc("test_util/get_about_locations") result = list(util.get_about_locations(location)) - expected = 'get_about_locations/about.ABOUT' + expected = "get_about_locations/about.ABOUT" assert result[0].endswith(expected) # FIXME: these are not very long/deep paths def test_get_locations_with_very_long_path(self): longpath = ( - 'longpath' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' + "longpath" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" ) - test_loc = extract_test_loc('test_util/longpath.zip') + test_loc = extract_test_loc("test_util/longpath.zip") result = list(util.get_locations(test_loc)) assert any(longpath in r for r in result) class TestCsv(unittest.TestCase): - def test_load_csv_without_mapping(self): - test_file = get_test_loc('test_util/csv/about.csv') - expected = [dict([ - ('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) + test_file = get_test_loc("test_util/csv/about.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) ] result = util.load_csv(test_file) assert expected == result def test_load_csv_load_rows(self): - test_file = get_test_loc('test_util/csv/about.csv') - expected = [dict([ - ('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) + test_file = get_test_loc("test_util/csv/about.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) ] result = util.load_csv(test_file) assert expected == result def test_load_csv_does_convert_column_names_to_lowercase(self): - test_file = get_test_loc('test_util/csv/about_key_with_upper_case.csv') - expected = [dict( - [('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) - ] + test_file = get_test_loc("test_util/csv/about_key_with_upper_case.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) + ] result = util.load_csv(test_file) assert expected == result def test_format_about_dict_output(self): - about = [dict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', [u'test.c']), - (u'name', u'AboutCode-toolkit'), - (u'license_expression', u'mit AND bsd-new'), - (u'license_key', [u'mit', u'bsd-new'])])] - - expected = [dict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', u'test.c'), - (u'name', u'AboutCode-toolkit'), - (u'license_expression', u'mit AND bsd-new'), - (u'license_key', u'mit\nbsd-new')])] + about = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", ["test.c"]), + ("name", "AboutCode-toolkit"), + ("license_expression", "mit AND bsd-new"), + ("license_key", ["mit", "bsd-new"]), + ] + ) + ] + + expected = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", "test.c"), + ("name", "AboutCode-toolkit"), + ("license_expression", "mit AND bsd-new"), + ("license_key", "mit\nbsd-new"), + ] + ) + ] output = util.format_about_dict_output(about) assert output == expected def test_load_csv_microsoft_utf_8(self): - test_file = get_test_loc('test_util/csv/test_ms_utf8.csv') - expected = [ - dict([(u'about_resource', u'/myFile'), (u'name', u'myName')])] + test_file = get_test_loc("test_util/csv/test_ms_utf8.csv") + expected = [dict([("about_resource", "/myFile"), ("name", "myName")])] result = util.load_csv(test_file) assert expected == result def test_load_csv_utf_8(self): - test_file = get_test_loc('test_util/csv/test_utf8.csv') - expected = [ - dict([(u'about_resource', u'/myFile'), (u'name', u'\u540d')])] + test_file = get_test_loc("test_util/csv/test_utf8.csv") + expected = [dict([("about_resource", "/myFile"), ("name", "\u540d")])] result = util.load_csv(test_file) assert expected == result class TestJson(unittest.TestCase): - def test_load_json(self): - test_file = get_test_loc('test_util/json/expected.json') - expected = [dict([ - ('about_file_path', '/load/this.ABOUT'), - ('about_resource', '.'), - ('name', 'AboutCode'), - ('version', '0.11.0')]) + test_file = get_test_loc("test_util/json/expected.json") + expected = [ + dict( + [ + ("about_file_path", "/load/this.ABOUT"), + ("about_resource", "."), + ("name", "AboutCode"), + ("version", "0.11.0"), + ] + ) ] result = util.load_json(test_file) assert expected == result def test_load_json_multi_entries(self): - test_file = get_test_loc('test_util/json/multi_entries.json') - expected = [dict([ - ('about_file_path', '/load/this.ABOUT'), - ('about_resource', '.'), - ('name', 'AboutCode'), - ('version', '0.11.0')]), - dict([ - ('about_file_path', '/load/that.ABOUT'), - ('about_resource', '.'), - ('name', 'that')]) + test_file = get_test_loc("test_util/json/multi_entries.json") + expected = [ + dict( + [ + ("about_file_path", "/load/this.ABOUT"), + ("about_resource", "."), + ("name", "AboutCode"), + ("version", "0.11.0"), + ] + ), + dict( + [("about_file_path", "/load/that.ABOUT"), ("about_resource", "."), ("name", "that")] + ), ] result = util.load_json(test_file) assert expected == result def test_load_json2(self): - test_file = get_test_loc('test_util/json/expected_need_mapping.json') - expected = [dict(dict([ - ('about_file', '/load/this.ABOUT'), - ('about_resource', '.'), - ('version', '0.11.0'), - ('name', 'AboutCode'), - ]) - )] + test_file = get_test_loc("test_util/json/expected_need_mapping.json") + expected = [ + dict( + dict( + [ + ("about_file", "/load/this.ABOUT"), + ("about_resource", "."), + ("version", "0.11.0"), + ("name", "AboutCode"), + ] + ) + ) + ] result = util.load_json(test_file) assert expected == result def test_load_non_list_json(self): - test_file = get_test_loc('test_util/json/not_a_list_need_mapping.json') - expected = [{ - 'path': '/load/this.ABOUT', - 'about_resource': '.', - 'name': 'AboutCode', - 'version': '0.11.0' - }] + test_file = get_test_loc("test_util/json/not_a_list_need_mapping.json") + expected = [ + { + "path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0", + } + ] result = util.load_json(test_file) assert expected == result def test_load_non_list_json2(self): - test_file = get_test_loc('test_util/json/not_a_list.json') - expected = [{ - 'about_file_path': '/load/this.ABOUT', - 'about_resource': '.', - 'name': 'AboutCode', - 'version': '0.11.0' - }] + test_file = get_test_loc("test_util/json/not_a_list.json") + expected = [ + { + "about_file_path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0", + } + ] result = util.load_json(test_file) assert expected == result def test_load_json_from_scancode(self): - test_file = get_test_loc('test_util/json/scancode_info.json') - expected = [{ - 'about_resource': 'lic.txt', - 'name': 'lic.txt', - 'type': 'file', - 'base_name': 'lic', - 'extension': '.txt', - 'size': 1463, - 'date': '2023-07-26', - 'sha1': 'bb3f381f9ec25416c0c3b4628f7f6b923ced040f', - 'md5': '63f9ec8c32874a5d987d78b9a730a6b8', - 'sha256': 'd71777b3dc333f540a871bf2ef6380e646a10f2ac1f077ce4f34326e16fb6995', - 'mime_type': 'text/plain', - 'file_type': 'ASCII text, with very long lines', - 'programming_language': None, - 'is_binary': False, - 'is_text': True, - 'is_archive': False, - 'is_media': False, - 'is_source': False, - 'is_script': False, - 'files_count': 0, - 'dirs_count': 0, - 'size_count': 0, - 'scan_errors': [] - }] + test_file = get_test_loc("test_util/json/scancode_info.json") + expected = [ + { + "about_resource": "lic.txt", + "name": "lic.txt", + "type": "file", + "base_name": "lic", + "extension": ".txt", + "size": 1463, + "date": "2023-07-26", + "sha1": "bb3f381f9ec25416c0c3b4628f7f6b923ced040f", + "md5": "63f9ec8c32874a5d987d78b9a730a6b8", + "sha256": "d71777b3dc333f540a871bf2ef6380e646a10f2ac1f077ce4f34326e16fb6995", + "mime_type": "text/plain", + "file_type": "ASCII text, with very long lines", + "programming_language": None, + "is_binary": False, + "is_text": True, + "is_archive": False, + "is_media": False, + "is_source": False, + "is_script": False, + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [], + } + ] result = util.load_scancode_json(test_file) assert expected == result def test_format_about_dict_for_json_output(self): - about = [dict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', dict([(u'test.c', None)])), - (u'name', u'AboutCode-toolkit'), - (u'license_key', [u'mit', u'bsd-new'])])] - - expected = [dict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', u'test.c'), - (u'name', u'AboutCode-toolkit'), - (u'licenses', [ - dict([(u'key', u'mit')]), - dict([(u'key', u'bsd-new')])])])] + about = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", dict([("test.c", None)])), + ("name", "AboutCode-toolkit"), + ("license_key", ["mit", "bsd-new"]), + ] + ) + ] + + expected = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", "test.c"), + ("name", "AboutCode-toolkit"), + ("licenses", [dict([("key", "mit")]), dict([("key", "bsd-new")])]), + ] + ) + ] output = util.format_about_dict_for_json_output(about) assert output == expected class TestMiscUtils(unittest.TestCase): - def test_load_yaml_about_file_with_no_dupe(self): - test = ''' + test = """ name: test license_expression: mit notes: dup key here - ''' + """ saneyaml.load(test, allow_duplicate_keys=False) def test_load_yaml_about_file_raise_exception_on__duplicate(self): - test = ''' + test = """ name: test notes: some notes notes: dup key here @@ -535,15 +597,15 @@ def test_load_yaml_about_file_raise_exception_on__duplicate(self): notes: dup key here license_expression: mit notes: dup key here - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except saneyaml.UnsupportedYamlFeatureError as e: - assert 'Duplicate key in YAML source: notes' == str(e) + assert "Duplicate key in YAML source: notes" == str(e) def test_load_yaml_about_file_raise_exception_on_invalid_yaml_ignore_non_key_line(self): - test = ''' + test = """ name: test - notes: some notes - notes: dup key here @@ -552,15 +614,15 @@ def test_load_yaml_about_file_raise_exception_on_invalid_yaml_ignore_non_key_lin notes: dup key here license_expression: mit notes dup key here - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass def test_load_yaml_about_file_with_multiline(self): - test = ''' + test = """ name: test owner: test notes: | @@ -570,38 +632,46 @@ def test_load_yaml_about_file_with_multiline(self): notes: continuation line description: sample - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except saneyaml.UnsupportedYamlFeatureError as e: # notes: exceptio is rasied only for the first dupe - assert 'Duplicate key in YAML source: owner' == str(e) + assert "Duplicate key in YAML source: owner" == str(e) def test_ungroup_licenses(self): about = [ - dict([ - (u'key', u'mit'), - (u'name', u'MIT License'), - (u'file', u'mit.LICENSE'), - (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit'), - (u'spdx_license_key', u'MIT')]), - dict([ - (u'key', u'bsd-new'), - (u'name', u'BSD-3-Clause'), - (u'file', u'bsd-new.LICENSE'), - (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new'), - (u'spdx_license_key', u'BSD-3-Clause')]) + dict( + [ + ("key", "mit"), + ("name", "MIT License"), + ("file", "mit.LICENSE"), + ("url", "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit"), + ("spdx_license_key", "MIT"), + ] + ), + dict( + [ + ("key", "bsd-new"), + ("name", "BSD-3-Clause"), + ("file", "bsd-new.LICENSE"), + ("url", "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new"), + ("spdx_license_key", "BSD-3-Clause"), + ] + ), ] - expected_lic_key = [u'mit', u'bsd-new'] - expected_lic_name = [u'MIT License', u'BSD-3-Clause'] - expected_lic_file = [u'mit.LICENSE', u'bsd-new.LICENSE'] + expected_lic_key = ["mit", "bsd-new"] + expected_lic_name = ["MIT License", "BSD-3-Clause"] + expected_lic_file = ["mit.LICENSE", "bsd-new.LICENSE"] expected_lic_url = [ - u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', - u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new'] - expected_spdx = [u'MIT', u'BSD-3-Clause'] - lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, _matched_text = util.ungroup_licenses( - about) + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new", + ] + expected_spdx = ["MIT", "BSD-3-Clause"] + lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, _matched_text = ( + util.ungroup_licenses(about) + ) assert expected_lic_key == lic_key assert expected_lic_name == lic_name assert expected_lic_file == lic_file @@ -609,23 +679,23 @@ def test_ungroup_licenses(self): assert expected_spdx == spdx_lic_key def test_unique_does_deduplicate_and_keep_ordering(self): - items = ['a', 'b', 'd', 'b', 'c', 'a'] - expected = ['a', 'b', 'd', 'c'] + items = ["a", "b", "d", "b", "c", "a"] + expected = ["a", "b", "d", "c"] results = util.unique(items) assert expected == results def test_unique_can_handle_About_object(self): - base_dir = 'some_dir' + base_dir = "some_dir" test = { - 'about_resource': '.', - 'author': '', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'custom_empty': '', - 'description': 'AboutCode is a tool\nfor files.', - 'license': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.' + "about_resource": ".", + "author": "", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "custom_empty": "", + "description": "AboutCode is a tool\nfor files.", + "license": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", } a = model.About() @@ -635,7 +705,7 @@ def test_unique_can_handle_About_object(self): c.load_dict(test, base_dir) b = model.About() - test.update(dict(about_resource='asdasdasd')) + test.update(dict(about_resource="asdasdasd")) b.load_dict(test, base_dir) abouts = [a, b] @@ -644,15 +714,18 @@ def test_unique_can_handle_About_object(self): def test_copy_license_notice_files(self): base_dir = get_temp_dir() - reference_dir = get_test_loc('test_util/licenses') - fields = [(u'license_expression', u'mit or public-domain'), - (u'about_resource', u'.'), - (u'name', u'test'), - (u'license_key', [u'mit', u'public-domain']), - (u'license_file', [u'mit.LICENSE, mit2.LICENSE', u'public-domain.LICENSE'])] - util.copy_license_notice_files(fields, base_dir, reference_dir, '') - licenses = ['mit.LICENSE', 'mit2.LICENSE', 'public-domain.LICENSE'] + reference_dir = get_test_loc("test_util/licenses") + fields = [ + ("license_expression", "mit or public-domain"), + ("about_resource", "."), + ("name", "test"), + ("license_key", ["mit", "public-domain"]), + ("license_file", ["mit.LICENSE, mit2.LICENSE", "public-domain.LICENSE"]), + ] + util.copy_license_notice_files(fields, base_dir, reference_dir, "") + licenses = ["mit.LICENSE", "mit2.LICENSE", "public-domain.LICENSE"] from os import listdir + copied_files = listdir(base_dir) assert len(licenses) == len(copied_files) for license in licenses: @@ -660,24 +733,26 @@ def test_copy_license_notice_files(self): def test_copy_file(self): des = get_temp_dir() - test_file = get_test_loc('test_util/licenses/mit.LICENSE') - licenses = ['mit.LICENSE'] + test_file = get_test_loc("test_util/licenses/mit.LICENSE") + licenses = ["mit.LICENSE"] err = util.copy_file(test_file, des) from os import listdir + copied_files = listdir(des) assert len(licenses) == len(copied_files) - assert err == '' + assert err == "" for license in licenses: assert license in copied_files def test_copy_file_with_dir(self): des = get_temp_dir() - test_dir = get_test_loc('test_util/licenses/') - licenses = ['mit.LICENSE', 'mit2.LICENSE', 'public-domain.LICENSE'] + test_dir = get_test_loc("test_util/licenses/") + licenses = ["mit.LICENSE", "mit2.LICENSE", "public-domain.LICENSE"] err = util.copy_file(test_dir, des) - assert err == '' + assert err == "" import os + files_list = [] dir_list = [] # Get the directories and files in the 'des' recursively @@ -693,15 +768,15 @@ def test_copy_file_with_dir(self): assert license in files_list def test_strip_inventory_value(self): - test = [{'about_resource': 'empty_newlines.rpm\n\n', 'name': 'empty_newlines.rpm'}, - {'about_resource': 'spaces_after.rpm ', - 'name': 'spaces_after.rpm '}, - {'about_resource': 'value_after_newline\n123.rpm ', - 'name': 'value_after'}] - expected = [{'about_resource': 'empty_newlines.rpm', 'name': 'empty_newlines.rpm'}, - {'about_resource': 'spaces_after.rpm', - 'name': 'spaces_after.rpm'}, - {'about_resource': 'value_after_newline\n123.rpm', - 'name': 'value_after'}] + test = [ + {"about_resource": "empty_newlines.rpm\n\n", "name": "empty_newlines.rpm"}, + {"about_resource": "spaces_after.rpm ", "name": "spaces_after.rpm "}, + {"about_resource": "value_after_newline\n123.rpm ", "name": "value_after"}, + ] + expected = [ + {"about_resource": "empty_newlines.rpm", "name": "empty_newlines.rpm"}, + {"about_resource": "spaces_after.rpm", "name": "spaces_after.rpm"}, + {"about_resource": "value_after_newline\n123.rpm", "name": "value_after"}, + ] stripped_result = util.strip_inventory_value(test) assert stripped_result == expected diff --git a/tests/testing_utils.py b/tests/testing_utils.py index d8c0d43c..4b0764ff 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -32,9 +32,9 @@ handler.setLevel(logging.DEBUG) logger.addHandler(handler) -TESTDATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'testdata') +TESTDATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "testdata") -on_windows = 'win32' in sys.platform +on_windows = "win32" in sys.platform on_posix = not on_windows @@ -58,11 +58,10 @@ def create_dir(location): """ if not os.path.exists(location): os.makedirs(location) - os.chmod(location, stat.S_IRWXU | stat.S_IRWXG - | stat.S_IROTH | stat.S_IXOTH) + os.chmod(location, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) -def build_temp_dir(prefix='test-attributecode-'): +def build_temp_dir(prefix="test-attributecode-"): """ Create and return a new unique empty directory created in base_dir. """ @@ -71,7 +70,7 @@ def build_temp_dir(prefix='test-attributecode-'): return location -def get_temp_file(file_name='test-attributecode-tempfile'): +def get_temp_file(file_name="test-attributecode-tempfile"): """ Return a unique new temporary file location to a non-existing temporary file that can safely be created without a risk of name @@ -103,7 +102,7 @@ def extract_zip(location, target_dir): Extract a zip archive file at location in the target_dir directory. """ if not os.path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: for info in zipf.infolist(): @@ -118,7 +117,7 @@ def extract_zip(location, target_dir): if not os.path.exists(target): os.makedirs(target) if not os.path.exists(target): - with open(target, 'wb') as f: + with open(target, "wb") as f: f.write(content) @@ -144,30 +143,27 @@ def run_about_command_test(options, expected_rc=0): On success, return stdout and stderr. """ root_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - about_cmd = os.path.join(root_dir, 'about') + about_cmd = os.path.join(root_dir, "about") args = [about_cmd] + options about = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True if on_windows else False) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True if on_windows else False + ) stdout, stderr = about.communicate() rc = about.poll() if rc != expected_rc: - opts = ' '.join(args) + opts = " ".join(args) error = ( - 'Failure to run command: %(opts)s\n' - 'stdout:\n' - '{stdout}\n' - '\n' - 'stderr:\n' - '{stderr}\n' + "Failure to run command: %(opts)s\nstdout:\n{stdout}\n\nstderr:\n{stderr}\n" ).format(**locals()) assert rc == expected_rc, error return stdout, stderr -def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): +def run_about_command_test_click( + options, + expected_rc=0, + monkeypatch=None, +): """ Run an "about" command as a Click-controlled subprocess with the `options` list of options. Return a click.testing.Result object. @@ -177,9 +173,17 @@ def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): import click from click.testing import CliRunner from attributecode import cmd + if monkeypatch: - monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) - monkeypatch.setattr(click , 'get_terminal_size', lambda : (80, 43,)) + monkeypatch.setattr(click._termui_impl, "isatty", lambda _: True) + monkeypatch.setattr( + click, + "get_terminal_size", + lambda: ( + 80, + 43, + ), + ) runner = CliRunner() result = runner.invoke(cmd.about, options, catch_exceptions=False) @@ -187,20 +191,20 @@ def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): output = result.output if result.exit_code != expected_rc: opts = get_opts(options) - error = ''' + error = """ Failure to run: about %(opts)s output: %(output)s -''' % locals() +""" % locals() assert result.exit_code == expected_rc, error return result def get_opts(options): try: - return ' '.join(options) + return " ".join(options) except: try: - return b' '.join(options) + return b" ".join(options) except: - return b' '.join(map(repr, options)) + return b" ".join(map(repr, options))