diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..96c89ceb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Ignore all Git auto CR/LF line endings conversions +* -text +pyproject.toml export-subst diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 00000000..8d8aa551 --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,32 @@ +name: CI Documentation + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-24.04 + + strategy: + max-parallel: 4 + matrix: + python-version: [3.13] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install Dependencies + run: ./configure --dev + + - name: Check documentation and HTML for errors and dead links + run: make docs-check + + - name: Check documentation for style errors + run: make doc8 + + diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml new file mode 100644 index 00000000..1ae19d7f --- /dev/null +++ b/.github/workflows/pypi-release.yml @@ -0,0 +1,88 @@ +name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch + + +# This is executed automatically on a tag in the main branch + +# Summary of the steps: +# - build wheels and sdist +# - upload wheels and sdist to PyPI +# - create gh-release and upload wheels and dists there +# TODO: smoke test wheels and sdist +# TODO: add changelog to release text body + +# WARNING: this is designed only for packages building as pure Python wheels + +on: + workflow_dispatch: + push: + tags: + - "v*.*.*" + +jobs: + build-pypi-distribs: + name: Build and publish library to PyPI + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install pypa/build and twine + run: python -m pip install --user --upgrade build twine pkginfo + + - name: Build a binary wheel and a source tarball + run: python -m build --wheel --sdist --outdir dist/ + + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + + - name: Upload built archives + uses: actions/upload-artifact@v4 + with: + name: pypi_archives + path: dist/* + + + create-gh-release: + name: Create GH release + needs: + - build-pypi-distribs + runs-on: ubuntu-24.04 + + steps: + - name: Download built archives + uses: actions/download-artifact@v4 + with: + name: pypi_archives + path: dist + + - name: Create GH release + uses: softprops/action-gh-release@v2 + with: + draft: true + files: dist/* + + + create-pypi-release: + name: Create PyPI release + needs: + - create-gh-release + runs-on: ubuntu-24.04 + environment: pypi-publish + permissions: + id-token: write + + steps: + - name: Download built archives + uses: actions/download-artifact@v4 + with: + name: pypi_archives + path: dist + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7b5906d7..601928fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +# Python compiled files *.py[cod] # virtualenv and other misc bits +/src/*.egg-info *.egg-info /dist /build @@ -11,11 +13,16 @@ /Lib /pip-selfcheck.json /tmp +/venv .Python /include /Include /local -/man/ +*/local/* +/local/ +/share/ +/tcl/ +/.eggs/ # Installer logs pip-log.txt @@ -34,10 +41,20 @@ htmlcov .project .pydevproject .idea +org.eclipse.core.resources.prefs +.vscode +.vs # Sphinx docs/_build +docs/bin +docs/build +docs/include +docs/Lib +doc/pyvenv.cfg +pyvenv.cfg +# Various junk and temp files .DS_Store *~ .*.sw[po] @@ -45,7 +62,16 @@ docs/_build .ve *.bak /.cache/ -/.settings/ -/tcl/ + +# pyenv /.python-version -/.tox/ +/man/ +/.pytest_cache/ +lib64 +tcl + +# Ignore Jupyter Notebook related temp files +.ipynb_checkpoints/ +/virtualenv.pyz +/.ruff_cache/ +.env diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 00000000..27c15959 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,29 @@ +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Build in latest ubuntu/python +build: + os: ubuntu-22.04 + tools: + python: "3.13" + +# Build PDF & ePub +formats: + - epub + - pdf + +# Where the Sphinx conf.py file is located +sphinx: + configuration: docs/source/conf.py + +# Setting the python version and doc build requirements +python: + install: + - method: pip + path: . + extra_requirements: + - dev diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 91d9131d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,22 +0,0 @@ -language: python - -python: - - "2.7" - - "3.6" - -install: - - ./configure etc/conf/dev - -script: - - bin/py.test -vvs - -notifications: - irc: - channels: - - "chat.freenode.net#aboutcode" - on_success: change - on_failure: always - use_notice: true - skip_join: true - template: - - "%{repository_slug}#%{build_number} (%{branch} - %{commit} : %{author}): %{message} : %{build_url}" diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 00000000..d28e17fb --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,3 @@ +The following organizations or individuals have contributed to this repo: + +-nexB, Inc. diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 00000000..1e518f0e --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,457 @@ +============================== +Changelog +============================== + +2025-xx-xx + Release xx + + * Updated docs format + +2025-03-31 + Release 11.1.1 + + * Deleted unused files + * Updated AppVeyor configuration to use Python 3.9 + * Modified the PyPI release script to utilize upload-artifact@v4 + + +2025-03-28 + Release 11.1.0 + + * Drop support for python version earlier than 3.9 + * Add ability to "exclude" path in the check and inventory #583 + * Add support for the special '-' FILE to print to on screen/to stdout + in inventory #584 + + +2024-09-16 + Release 11.0.2 + + * Fixed the installation issues with docker (#571, #572) + + +2024-08-06 + Release 11.0.1 + + * Update doc formatting + * Added fields type and types description into the spec + * Update link references of ownership from nexB to aboutcode-org + + +2024-07-15 + Release 11.0.0 + + * Add character support (at most 2 characters) for `attribute` field + * Strip empty newline characters when loading an inventory + * Catch invalid license_expression + * Update the specification to 3.3.2 + * Support "declared_license_expression" and "other_license_expression" + * Updated "about_resource" to be an optional field + * Updated spec to v4.0.0 as moving `about_resource` from mandatory to + optional + + +2023-09-25 + Release 10.1.0 + + * Fixed `transform` with nested list #531 + * Added curl dependency in Dockerfile #532 + * Introduce spdx_license_expression + * Ability to transform spdx license key from spdx_license_expression to + license_expression (i.e. Generate attribution with + spdx_license_expression) #513 + * Ability to configure the proxy settings #533 + * Fixed licenses issue #534 + +2023-08-20 + Release 10.0.0 + + * Fixd error in load_json in util.py + * Code cleanup + * Work with the SCTK version 32 and later (Drop support for SCTK + version 31 and earlier) + * Implement `--scancode` option for `gen` + + +2023-07-14 + Release 9.0.0 + + * The tool will now show which worksheet (if .xlsx input) is the tool + working on + * Error handling if defined worksheet does not exist + * Adopt 3.3.1 specification: introduce ``ignored_resources`` + + +2023-03-09 + Release 8.0.0 + + * Fixed the transform code for xlsx and json + * Remove irrelevant error for attrib + * Add support to identify worksheet name for XLSX input + * The severity error level for "contains illegal name characters (or + empty spaces) and is ignored" is changed from ERROR to WARNING + * Remove the limitation to ASCII only + * Drop support for python3.6 + * Update valid chatacters for file/path name + * Adopt 3.3.0 specification + + +2022-10-24 + Release 7.2.0 + + * Add option to validate `license_expression` in the `check` command + + +2022-10-24 + Release 7.1.1 + + * This new release has no feature changes. + + +2022-10-24 + Release 7.1.0 + + * Fixed version mismatch + (https://github.com/nexB/aboutcode-toolkit/issues/510) + * Improve `check` performance + (https://github.com/nexB/aboutcode-toolkit/issues/511) + * Relax the requirement to have the same format for input and output + for `transform` + * Collect and handle the "matched_text" from the `--license-text` + option from the scancode-toolkit + + +2022-03-21 + Release 7.0.2 + + * Relax dependency constraints + * Use latest skeleton and settings + + +2022-03-21 + Release 7.0.1 + + * Bump openpyxl to 3.0.9 + * Better handling of invalid API URL + * Better formatting for default HTML template + +2022-03-01 + Release 7.0.0 + + * Add '@' as a supported character for filename #451 + * Add support to collect redistributable sources #22 + * Handle trailing spaces in field names during `transform` #456 + * Remove restriction of python27 only on windows #453 + * Documentation updated + * Code enhancement + * Remove thirdparty/ + * Update configuration scripts + * Use readthedocs for documentation + * Add Dockerfile to run aboutcode with docker + * Add new option to choose extract license from ScanCode LicenseDB or + DJC License Library + * Add ability to transform XLSX file + * Support XLSX file format for `inventory`, `gen` and `attrib` + * Add 'spdx_license_key' support + * Add option to save error log in `check` command + * New `gen_license` option + * Bump PyYAML to 6.0 + * Add '%" as a supported character + * Update default template + * All errors are logged if and only if the `verbose` option is set. + Otherwise, ony 'Critical' and 'Warning' errors will be showed/logged + * Ability to generate attribution notice directly from an input + inventory + * Remove the restriction of requiring 'about_resource' field in the + input if performing `attrib` from an inventory + +2021-04-02 + Release 6.0.0 + + * This new release has no feature changes. + * It has relaxed PyYAML dependencies and drop Python 2 support + +2020-09-01 + Release 5.1.0 + + * Add support for `package_url` #396 + * Fixed #443 and #444 issue with multiple licenses/license_files + * Fixed #442 no special characters allowed for `license_key`, + `license_name` and `license_expression` + * Fixed #446 Better error handling + +2020-08-11 + Release 5.0.0 + + * Enhance the `transform` to also work with JSON file + * Update transform code (See #427 and #428) + * Fixed #431 - Error handling for empty "_file" fields + * Fixed #432 - Handled UTF-8 variant invented by Microsoft + * Fixed #433 - problem was caused by the different multi-lic file + format between json and CSV (CSV with '\n' line break) + * Fixed #436 - issue about copy with the `--reference` option + * Fixed #396 - support for alternative output for Android + +2020-05-05 + Release 4.0.2 + + * Upgrade license-expression library to v1.2 + * Fix the missing `multi_sort` filter for Jinja2 + * Update help text for `--vartext` + +2019-10-17 + Release 4.0.1 + + * Declare to follow SemVer for versioning schema + * Update REFERENCE.rst and README.rst + * Update license-expression library + + +2019-10-09 + + Release 4.0.0 + + * Support filenames/path with special characters #310 #378 #392 + * Update ABOUT file format to match the specification + * Log version of which AbcTK was used #397 + * Fix the licenses (key, name, file) not in sync issue #406 + * Correct invalid msg for boolean fields #403 + * Remove the `about_file_path` key/column from input/output #364 + * Use ',' to support multiple files #404 + * Fix bugs in `transform` #408, #412 + +2018-11-15 + + Release 3.3.0 + + * Update the list of common license keys + * New UrlListField introduced for list of urls + * The UrlField is now only taking single URL value + * The owner is now a StringField instead of ListField + * Format the ordering of the generated ABOUT file (See + https://github.com/nexB/aboutcode-toolkit/issues/349#issuecomment-438871444) + * '+' and '(' and ')' is now supported in license_expression + * The key 'about_resource_path' is removed + * Revert back the requirement of the 'name' field + * Add a new supported 'internal_use_only' key + +2018-10-23 + + Release 3.2.2 + + * Fix the version number + * `name` field is no longer a required field + +2018-10-22 + + Release 3.2.1 + + * The 'license' field is now become 'license_key' + * Multiple licenses support + * No support for referenceing multuiple resources + * Fix the incorrect boolean field behaviors + * Display number of errors/warnings in the error log + +2018-09-19 + + Release 3.1.3 + + * Minor update + +2018-09-19 + + Release 3.1.2 + + * New `--vartext` option for `attrib` + * Add support for `checksum_sha256` and `author_file` + * `check` command will not count INFO message as error when `--verbose` + is set + * Update `track_change` to `track_changes` + * New `--filter` and `--mapping-output` options for `inventory` + +2018-6-25 + + Release 3.1.1 + + * No support of multiple occurrence keys in the input + * Updated the specification document + * Fixed bug that cause template processing error in attrib + + etc... + + +2018-6-8 Chin-Yeung Li + + Release 3.1.0 + + * Fixed JSON input from AboutCode manger export and ScanCode output + * Added a new option `mapping-file` to support using a custom file for + mapping + * Change the name of the option `--show-all` to `--verbose` + * Better error handling for copying file with permission issue + * Support timestamp in attribution output + + etc... + + +2017-11-17 Chin-Yeung Li + + Release 3.0.* + + ABOUT files is now YAML formatted. Supported license expression. + Supported JSON input and output format: + https://github.com/nexB/aboutcode-toolkit/issues/246 and + https://github.com/nexB/aboutcode-toolkit/issues/277 Support Python 3: + https://github.com/nexB/aboutcode-toolkit/issues/280 Refined help texts + Refined USAGE texts Refined SPECs + + Input key changes: + + `about_file` is replaced by `about_file_path` `dje_license_key` is + replaced by `license_expression` `version` is no longer a required + field `home_url` is now `homepage_url` + + API Updated: + + - Break down the 3 major functions: `inventory`, `genabout` and + `genattrib` into 3 subcommands: + i.e. `about inventory`, `about generate` and `about attrib` + + - A new `check` subcommand: + https://github.com/nexB/aboutcode-toolkit/issues/281 + + - Some options changes + `--extract_license` becomes `--fetch-license` + `--license_text_location` becomes `--license-notice-text-location` + + etc... + + +2016-06-27 Chin-Yeung Li + + Release 2.3.2 + + * Documentation updates + + +2016-03-29 Philippe Ombredanne + + Release 2.3.1 + + * Various minor bug fixes and improvements + * Support for the latest DejaCode API if you use DejaCode + + +2016-03-28 Philippe Ombredanne + + Release 2.3.0 + + * Various minor bug fixes and improvements + * Support for the latest DejaCode API if you use DejaCode + + +2015-10-23 Chin-Yeung Li + + Release 2.2.0 + + * Improved CLI error messages + * Fixed the filtering of dicts with empty values. + * Refined help texts + * Updated configure script + * Refactorings and code simplifications + * Fixed misleading error message when using invalid api_url + + +2015-10-09 Chin-Yeung Li + + Release 2.1.0 + + * Minor code refactoring + * Handle long path error on Windows OS when using genattrib with a zip + + +2015-09-29 Chin-Yeung Li + + Release 2.0.4 + + * Added support to run genattrib with a zip file and tests + * Display a "Completed" message once the generation is completed + + +2015-08-01 Chin-Yeung Li + + Release 2.0.3 + + * Fix the bug of using genattrib.py on Windows OS + * Display version when running about.py, genabout.py and genattrib.py + + +2015-07-07 Chin-Yeung Li + + Release 2.0.2 + + * Handle input's encoding issues + * Better error handling + * Writing to and reading from Windows OS with paths > 255 chars + + +2015-06-09 Chin-Yeung Li + + Release 2.0.1 + + * Configuration script fixes and updates basic documentation. + + +2015-05-05 Chin-Yeung Li + + Release 2.0.0 + + * Breaking API changes: + + * the dje_license field has been renamed to dje_license_key + * when a dje_license_key is present, a new dje_license_url will be + reported when fetching data from the DejaCode API. + * In genabout, the '--all_in_one' command line option has been + removed. It was not well specified and did not work as advertised. + + * in genattrib: + + * the Component List is now optional. + * there is a new experimental '--verification_location' command line + option. This option will be removed in the future version. Do not + use it. + * the '+' character is now supported in file names. + * several bugs have been fixed. + * error handling and error and warning reporting have been improved. + * New documentation in doc: + UsingAboutCodetoDocumentYourSoftwareAssets.pdf + + +2014-11-05 Philippe Ombredanne + + Release 1.0.2 + + * Minor bug fixes and improved error reporting. + + +2014-11-03 Philippe Ombredanne + + Release 1.0.1 + + * Minor bug fixes, such as extraneous debug printouts. + + +2014-10-31 Philippe Ombredanne + + Release 1.0.0 + + * Some changes in the spec, such as supporting only text in external + files. + * Several refinements including support for common licenses. + +2014-06-24 Chin-Yeung Li + + Release 0.8.1 + + * Initial release with minimal capabilities to read and validate ABOUT + files format 0.8.0 and output a CSV inventory. diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 00000000..590ba198 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,86 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, gender identity and +expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +Our Standards +------------- + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Our Responsibilities +-------------------- + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +Scope +----- + +This Code of Conduct applies both within project spaces and in public +spaces when an individual is representing the project or its community. +Examples of representing a project or community include using an +official project e-mail address, posting via an official social media +account, or acting as an appointed representative at an online or +offline event. Representation of a project may be further defined and +clarified by project maintainers. + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at pombredanne@gmail.com +or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . +All complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project’s leadership. + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant`_ , +version 1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +.. _Contributor Covenant: https://www.contributor-covenant.org diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f14ddf86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/aboutcode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +FROM python:3.9-slim-buster + +RUN apt-get update \ + && apt-get install -y bash bzip2 xz-utils zlib1g libxml2-dev libxslt1-dev libgomp1 libpopt0 curl\ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Create directory for aboutcode sources +RUN mkdir aboutcode-toolkit + +# Copy sources into docker container +COPY . aboutcode-toolkit + +# Set workdir +WORKDIR aboutcode-toolkit + +RUN bash -c "source ./configure" + +# Add aboutcode to path +#ENV PATH=$HOME/aboutcode-toolkit:$PATH + +# Set entrypoint to be the aboutcode command, allows to run the generated docker image directly with the aboutcode arguments: +# `docker run (...) ` +# Example: docker run --rm --name "aboutcode" -v ${PWD}:/project -v /tmp/result:/result aboutcode-toolkit attrib /project /result/c.html +ENTRYPOINT ["./about"] diff --git a/MANIFEST.in b/MANIFEST.in index 3fd2b3fb..0f197075 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,29 +1,25 @@ graft src -graft tests graft docs graft etc -graft example -graft thirdparty -prune src/aboutcode_toolkit.egg-info - -include about.ABOUT -include about -include about.bat -include about.cfg -include apache-2.0.LICENSE -include about.bat -include apache-2.0.LICENSE -include configure -include configure.bat -include MANIFEST.in +include *.LICENSE include NOTICE -include README.rst -include setup.cfg -include setup.py -include SPEC -include USAGE.rst -include .travis.yml -include appveyor.yml +include *.ABOUT +include *.toml +include *.yml +include *.rst +include *.png +include setup.* +include configure* +include requirements* +include .dockerignore +include .gitignore +include .readthedocs.yml +include manage.py +include Dockerfile* +include Makefile +include MANIFEST.in + +include .VERSION global-exclude *.py[co] __pycache__ *.*~ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..3041547b --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +# Python version can be specified with `$ PYTHON_EXE=python3.x make conf` +PYTHON_EXE?=python3 +VENV=venv +ACTIVATE?=. ${VENV}/bin/activate; + + +conf: + @echo "-> Install dependencies" + ./configure + +dev: + @echo "-> Configure and install development dependencies" + ./configure --dev + +doc8: + @echo "-> Run doc8 validation" + @${ACTIVATE} doc8 --quiet docs/ *.rst + +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix + +check: + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ + +clean: + @echo "-> Clean the Python env" + ./configure --clean + +test: + @echo "-> Run the test suite" + ${VENV}/bin/pytest -vvs + +docs: + rm -rf docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ + +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/NOTICE b/NOTICE index 0ed33771..cbdaef79 100644 --- a/NOTICE +++ b/NOTICE @@ -1,10 +1,19 @@ - Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +# +# Copyright (c) nexB Inc. and others. +# SPDX-License-Identifier: Apache-2.0 +# +# Visit https://aboutcode.org and https://github.com/aboutcode-org/ for support and download. +# ScanCode is a trademark of nexB Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/README.rst b/README.rst index d7db21b2..8a1a0b46 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,4 @@ +================= AboutCode Toolkit ================= @@ -20,8 +21,8 @@ In addition, this tool is able to generate attribution notices and identify redistributable source code used in your project to help you comply with open source licenses conditions. -This version of the AboutCode Toolkit follows the ABOUT specification version 3.1.4 at: -https://github.com/nexB/aboutcode-toolkit/blob/develop/SPECIFICATION.rst +This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.2 at: +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/specification.html Build and tests status @@ -38,7 +39,7 @@ Build and tests status REQUIREMENTS ------------ -The AboutCode Toolkit is tested with Python 2.7 and 3.6 on Linux, Mac and Windows. +The AboutCode Toolkit is tested with Python 3.9 or above only on Linux, Mac and Windows. You will need to install a Python interpreter if you do not have one already installed. @@ -47,27 +48,46 @@ version may be pre-installed, open a terminal and type: python --version +Note +~~~~ + Debian has decided that distutils is not a core python package, so it + is not included in the last versions of debian and debian-based OSes. + A solution is to run: `sudo apt install python3-distutils` + On Windows or Mac, you can download the latest Python here: https://www.python.org/downloads/ Download the .msi installer for Windows or the .dmg archive for Mac. Open and run the installer using all the default options. - INSTALLATION ------------ Checkout or download and extract the AboutCode Toolkit from: https://github.com/nexB/aboutcode-toolkit/ To install all the needed dependencies in a virtualenv, run (on posix): - source configure + ./configure or on windows: configure +ACTIVATE the VIRTUALENV +----------------------- +To activate the virtualenv, run (on posix): + source venv/bin/activate +or on windows: + venv\\bin\\activate + + +DEACTIVATE the VIRTUALENV +------------------------- +To deactivate the virtualenv, run (on both posix and windows): + deactivate + VERSIONING SCHEMA ----------------- -Starting at AboutCode version 4.0.0, the AboutCode Toolkit will follow SemVer for the versioning schema. +Starting at AboutCode version 4.0.0, the AboutCode Toolkit will follow +SemVer for the versioning schema. i.e. MAJOR.MINOR.PATCH format 1. MAJOR version when making incompatible API changes, @@ -77,19 +97,30 @@ i.e. MAJOR.MINOR.PATCH format REFERENCE --------- -See https://github.com/nexB/aboutcode-toolkit/blob/master/REFERENCE.rst for reference -on aboutcode-toolkit usage. +See https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/ +for documentation. +See +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/reference.html +for reference. TESTS and DEVELOPMENT --------------------- To install all the needed development dependencies, run (on posix): - source configure etc/conf/dev + ./configure --dev or on windows: - configure etc/conf/dev + configure --dev To verify that everything works fine you can run the test suite with: - py.test + pytest + + +CLEAN BUILD AND INSTALLED FILES +------------------------------- +To clean the built and installed files, run (on posix): + ./configure --clean +or on windows: + configure --clean HELP and SUPPORT @@ -112,7 +143,7 @@ The AboutCode Toolkit is available through GitHub. For the latest version visit: HACKING ------- We accept pull requests provided under the same license as this tool. -You agree to the http://developercertificate.org/ +You agree to the http://developercertificate.org/ LICENSE diff --git a/REFERENCE.rst b/REFERENCE.rst deleted file mode 100644 index d72fbd36..00000000 --- a/REFERENCE.rst +++ /dev/null @@ -1,339 +0,0 @@ -about -===== - -**Syntax** - -:: - - about [OPTIONS] [COMMANDS] - -**Options:** - -:: - - --version Show the version and exit. - -h, --help Show this message and exit. - -**Commands:** - -:: - - attrib LOCATION: directory, OUTPUT: output file - check LOCATION: directory - gen LOCATION: input file, OUTPUT: directory - inventory LOCATION: directory, OUTPUT: csv file - transform LOCATION: csv file, OUTPUT: csv file - - -attrib -====== - -**Syntax** - -:: - - about attrib [OPTIONS] LOCATION OUTPUT - - LOCATION: Path to an ABOUT file or a directory containing ABOUT files. - OUTPUT: Path to output file to write the attribution to. - -**Options:** - -:: - - --template PATH Path to a custom attribution template. - --vartext = Variable text as key=value for use in a custom attribution template. - --verbose Show all the errors and warning. - -q, --quiet Do not print any error/warning. - -h, --help Show this message and exit. - -Purpose -------- -Generate an attribution file which contains the all license information -from the LOCATION along with the license text. - -Assume the following: - -:: - - '/home/about_files/'** contains all the ABOUT files [LOCATION] - '/home/attribution/attribution.html' is the user's output path [OUTPUT] - -:: - - $ about attrib /home/about_files/ /home/attribution/attribution.html - -Options -------- - -:: - - --template - - This option allows you to use your own template for attribution generation. - For instance, if you have a custom template located at: - /home/custom_template/template.html - - $ about attrib --template /home/custom_template/template.html LOCATION OUTPUT - - --vartext - - This option allow you to pass variable texts to the attribution template - - $ about attrib --vartext "title=Attribution Notice" --vartext "header=Product 101" LOCATION OUTPUT - - Users can use the following in the template to get the vartext: - {{ variables['title'] }} - {{ variables['header'] }} - - --verbose - - This option tells the tool to show all errors found. - The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' - - -The following data are passed to jinja2 and, therefore, can be used for a custom template: - * about object: the about objects - * common_licenses: a common license keys list in licenses.py - * license_key_and_context: a dictionary list with license_key as a key and license text as the value - * license_file_name_and_key: a dictionary list with license file name as a key and license key as the value - * license_key_to_license_name: a dictionary list with license key as a key and license file name as the value - - -check -===== - -**Syntax** - -:: - - about check [OPTIONS] LOCATION - - LOCATION: Path to an ABOUT file or a directory with ABOUT files. - -**Options:** - -:: - - --verbose Show all the errors and warning - -h, --help Show this message and exit. - -Purpose -------- -Validating ABOUT files at LOCATION. - -Options -------- - -:: - - --verbose - - This option tells the tool to show all errors found. - The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' - - $ about check --verbose /home/project/about_files/ - - -gen -=== - -**Syntax** - -:: - - about gen [OPTIONS] LOCATION OUTPUT - - LOCATION: Path to a JSON or CSV inventory file. - OUTPUT: Path to a directory where ABOUT files are generated. - -**Options:** - -:: - - --fetch-license api_url api_key Fetch licenses data from DejaCode License - Library and create .LICENSE - side-by-side with the generated .ABOUT file. - The following additional options are required: - - api_url - URL to the DejaCode License Library - API endpoint - - api_key - DejaCode API key - Example syntax: - - about gen --fetch-license 'api_url' 'api_key' - --reference PATH Path to a directory with reference license - data and text files. - --verbose Show all the errors and warning. - -q, --quiet Do not print any error/warning. - -h, --help Show this message and exit. - -Purpose -------- -Given an inventory of ABOUT files at location, generate ABOUT files in base directory. - -Options -------- - -:: - - --fetch-license - - Fetch licenses text from a DejaCode API. and create .LICENSE side-by-side - with the generated .ABOUT file using data fetched from the DejaCode License Library. - - This option requires 2 parameters: - api_url - URL to the DJE License Library. - api_key - Hash key to authenticate yourself in the API. - - In addition, the input needs to have the 'license_expression' field. - (Please contact nexB to get the api_* value for this feature) - - $ about gen --fetch-license 'api_url' 'api_key' LOCATION OUTPUT - - --reference - - Copy the reference files such as 'license_files' and 'notice_files' to the - generated location from the specified directory. - - For instance, - the specified directory, /home/licenses_notices/, contains all the licenses and notices: - /home/licenses_notices/apache2.LICENSE - /home/licenses_notices/jquery.js.NOTICE - - $ about gen --license-notice-text-location /home/licenses_notices/ LOCATION OUTPUT - - --verbose - - This option tells the tool to show all errors found. - The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' - - -inventory -========= - -**Syntax** - -:: - - about inventory [OPTIONS] LOCATION OUTPUT - - LOCATION: Path to an ABOUT file or a directory with ABOUT files. - OUTPUT: Path to the JSON or CSV inventory file to create. - -**Options:** - -:: - - -f, --format [json|csv] Set OUTPUT file format. [default: csv] - --verbose Show all the errors and warning. - -q, --quiet Do not print any error/warning. - -h, --help Show this message and exit. - -Purpose -------- -Collect a JSON or CSV inventory of components from ABOUT files. - -Options -------- - -:: - - The above command will only inventory the ABOUT files which have the "license_expression: gpl-2.0" - - -f, --format [json|csv] - - Set OUTPUT file format. [default: csv] - - $ about inventory -f json LOCATION OUTPUT - - --verbose - - This option tells the tool to show all errors found. - The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' - - -Special Notes -============= -Multiple licenses support format --------------------------------- -The multiple licenses support format for CSV files are separated by line break - -+----------------+------+-----------------+----------------------+ -| about_resource | name | license_key | license_file | -+----------------+------+-----------------+----------------------+ -| test.tar.xz | test | | apache-2.0 | | apache-2.0.LICENSE | -| | | | mit | | mit.LICENSE | -+----------------+------+-----------------+----------------------+ - - -The multiple licenses support format for ABOUT files are by "grouping" with the keyword "licenses" - -:: - - about_resource: test.tar.xz - name: test - licenses: - - key: apache 2.0 - name: apache-2.0.LICENSE - - key: mit - name: mit.LICENSE - - -transform -========= - -**Syntax** - -:: - - about transform [OPTIONS] LOCATION OUTPUT - - LOCATION: Path to a CSV file. - OUTPUT: Path to CSV inventory file to create. - -**Options:** - -:: - - -c, --configuration FILE Path to an optional YAML configuration file. See - --help-format for format help. - --help-format Show configuration file format help and exit. - -q, --quiet Do not print error or warning messages. - --verbose Show all error and warning messages. - -h, --help Show this message and exit. - -Purpose -------- -Transform the CSV file at LOCATION by applying renamings, filters and checks and write a new CSV to OUTPUT. - -Options -------- - -:: - - -c, --configuration - - Path to an optional YAML configuration file. See--help-format for format help. - - $ about transform -c 'path to the YAML configuration file' LOCATION OUTPUT - - --help-format - - Show configuration file format help and exit. - This option will print out examples of the the YAML configuration file. - - Keys configuration are: `column_renamings`, `required_columns` and `column_filters` - - $ about transform --help-format - - --verbose - - This option tells the tool to show all errors found. - The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' - -Special Notes -============= -When using the `column_filters` configuration, all the standard required columns -(`about_resource` and `name`) and the user defined `required_columns` need to be included. diff --git a/SPECIFICATION.rst b/SPECIFICATION.rst deleted file mode 100644 index 152aac51..00000000 --- a/SPECIFICATION.rst +++ /dev/null @@ -1,416 +0,0 @@ -ABOUT File Specification v3.1.4 - - -Purpose -~~~~~~~ - -An ABOUT file provides a simple way to document the provenance (origin and -license) and other important or interesting information about a software -component. An ABOUT file is a small YAML formatted text file stored in the -codebase side-by-side with the software component file or archive that it -documents. No modification of the documented software is needed. - -The ABOUT format is plain text with field name/value pairs separated by a colon. -It is easy to read and create by hand and is designed first for humans, rather -than machines. The format is well-defined and structured just enough to make it -easy to process with software as well. It contains enough information to fulfill -key license requirements such as creating credits or attribution notices, -collecting redistributable source code, or providing information about new -versions of a software component. - - -Getting Started -~~~~~~~~~~~~~~~ - -A simple and valid ABOUT file named httpd-2.4.3.tar.gz.ABOUT may look like this:: - - about_resource: httpd-2.4.3.tar.gz - name: Apache HTTP Server - version: 2.4.3 - homepage_url: http://httpd.apache.org - download_url: http://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz - license_expression: apache-2.0 - licenses: - - key: apache-2.0 - name: Apache License 2.0 - file: apache-2.0.LICENSE - notice_file: httpd.NOTICE - copyright: Copyright (c) 2012 The Apache Software Foundation. - -The meaning of this ABOUT file is: - -- The file "httpd-2.4.3.tar.gz" is stored in the same directory and side-by-side - with the ABOUT file "httpd-2.4.3.tar.gz.ABOUT" that documents it. - -- The name of this component is "Apache HTTP Server" with version "2.4.3". - -- The home URL for this component is http://httpd.apache.org - -- The file "httpd-2.4.3.tar.gz" was originally downloaded from - http://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz - -- In the same directory, "apache-2.0.LICENSE" and "httpd.NOTICE" are files that - contain respectively the license text and the notice text for this component. - -- This component is licensed under "apache-2.0" - - -Specification -~~~~~~~~~~~~~ - -An ABOUT file is an ASCII YAML formatted text file. -Note that while Unicode characters are not supported in -an ABOUT file proper, external files can contain UTF-8 Unicode. -The key for the licenses field and the license_expression are dejacode license key. - - -ABOUT file name -~~~~~~~~~~~~~~~ - -An ABOUT file name can use a limited set of characters and is suffixed with a -".ABOUT" extension using any combination of uppercase and lowercase characters. - -A file name can contain only these US-ASCII characters: - -- digits from 0 to 9 -- uppercase and lowercase letters from A to Z -- the following symbols: "_", "-", "+", ".", "(", ")", "~", "[", "]", "{", "}" - -- The case of a file name is not significant. On case-sensitive file systems - (such as on Linux), a tool must report an error if two ABOUT files stored in - the same directory have the same lowercase file name. This is to ensure that - ABOUT files can be used across file systems. The convention is to use a - lowercase file name and an uppercase ABOUT extension. - - -Lines of text -~~~~~~~~~~~~~ - -An ABOUT file contains lines of US-ASCII text. Lines contain field names/values -pairs. The standard line ending is the LF character. The line ending characters -can be any LF, CR or CR/LF and tools must normalize line endings to LF when -processing an ABOUT file. Empty lines and lines containing only white spaces -that are not part of a field value continuation are ignored. Empty lines are -commonly used to improve the readability of an ABOUT file. - - -Field name -~~~~~~~~~~ - -A field name can contain only these US-ASCII characters: - -- digits from 0 to 9 -- uppercase and lowercase letters from A to Z -- the "_" underscore sign. - -- Field names are not case sensitive. For example, "HOMEPAGE_URL" and "HomePage_url" - represent the same field name. - -- A field name must start at the beginning of a new line. No spaces is allowed in the - field name. It can be followed by one or more spaces that must be ignored. - These spaces are commonly used to improve the readability of an ABOUT file. - - -Field value -~~~~~~~~~~~ - -The field value is separated from the field name by a ":" colon. The ":" colon -can be followed by one or more spaces that must be ignored. This also applies to -trailing white spaces: they must be ignored. - -The field value is composed of one or more lines of plain US-ASCII printable text. - -When a field value is a long string, additional continuation lines must start -with at least one space. In this case, the first space of an additional -continuation line is ignored and should be removed from the field value by tools. - -For instance:: - - description: This is a long description for a - software component that additional continuation line is used. - - -When a field value contains more than one line of text, a 'literal block' -(using |) is need. - -For instance:: - - description: | - This is a long description for a software component that spans - multiple lines with arbitrary line breaks. - - This text contains multiple lines. - - -Fields are mandatory, optional or custom extension -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A field can be mandatory, optional or custom extension. Tools -must report an error for missing mandatory fields. - - -Fields validation -~~~~~~~~~~~~~~~~~ - -When processing an ABOUT file, tools must report a warning or error if a field -is invalid. A field can be invalid for several reasons, such as invalid field -name syntax or invalid content. Tools should report additional validation error -details. The validation process should check that each field name is -syntactically correct and that fields contain correct values according to its -concise, common sense definition in this specification. For certain fields, -additional and specific validations are relevant such as URL validation, -path resolution and verification, and so forth. Tools should -report a warning for present fields that do not have any value. - - -Fields order and multiple occurrences -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The field order does not matter. Multiple occurrences of a field name is not -supported. - -The tool processing an ABOUT file or CSV/JSON input will issue an error when a -field name occurs more than once in the input file. - - -Field referencing a file -~~~~~~~~~~~~~~~~~~~~~~~~ - -The actual value of some fields may be contained in another file. This is useful -for long texts or to reference a common text in multiple ABOUT files such as a -common license text. In this case the field name is suffixed with "_file" and -the field value must be a path pointing to the file that contains the actual -value of the field. This path must be a POSIX path relative to the path of the -ABOUT file. The file content must be UTF-8-encoded text. This is in contrast -with field values contained directly in an ABOUT file that must be US-ASCII- -encoded text and allows to support non-ASCII text content. - -For example, the full license text for a component is often stored in a separate -file named COPYING:: - - licenses: - - file: linux.COPYING - -In this example, the README file is stored in a doc directory, one directory -above the ABOUT file directory, using a relative POSIX path:: - - licenses: - - file: ../docs/ruby.README - -In addition, there may be cases that a license can have 2 or more referenced license files. -If this is the case, a comma ',' is used to identify multiple files -For instance:: - - license_expression: gpl-2.0-plus - licenses: - - key: gpl-2.0-plus - file: COPYING, COPYING.LESSER - -Field referencing a URL -~~~~~~~~~~~~~~~~~~~~~~~ - -The value of a field may reference URLs such as a homepage or a download. In -this case the field name is suffixed with "_url" and the field value must be a -valid absolute URL starting with ftp://, http:// or https://. URLs are -informational and the content they may reference is ignored. For example, a -download URL is referenced this way:: - - download_url: http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.4.20.tar.bz2 - - -Flag fields -~~~~~~~~~~~ - -Flag fields have a "true" or "false" value. True, T, Yes, Y or x must be -interpreted as "true" in any case combination. False, F, No or N must be -interpreted as "false" in any case combination. - -Referencing the file or directory documented by an ABOUT file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -An ABOUT file documents one file or directory. The mandatory "about_resource" -field reference the documented file or directory. The value of the -"about_resource" field is the name or path of the referenced file or directory. - -A tool processing an ABOUT file must report an error if this field is missing. - -By convention, an ABOUT file is often stored in the same directory side-by-side -to the file or directory that it documents, but this is not mandatory. - -For example, a file named django.ABOUT contains the following field to document -the django-1.2.3.tar.gz archive stored in the same directory:: - - about_resource: django-1.2.3.tar.gz - -In this example, the ABOUT file documents a whole sub-directory:: - - about_resource: linux-kernel-2.6.23 - -In this example, the ABOUT file documents the current directory, using a "." -period to reference it:: - - about_resource: . - - -Other Mandatory fields -~~~~~~~~~~~~~~~~~~~~~~ - -When a tool processes an ABOUT file, it must issue an error if these mandatory -field are missing. - -- about_resource: The resource this file referencing to. -- name: Component name. - - -Optional Information fields -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- version: Component or package version. A component or package usually has a version, such as a - revision number or hash from a version control system (for a snapshot checked - out from VCS such as Subversion or Git). If not available, the version should - be the date the component was provisioned, in an ISO date format such as - 'YYYY-MM-DD'. - -- spec_version: The version of the ABOUT file format specification used for this - file. This is provided as a hint to readers and tools in order to support - future versions of this specification. - -- description: Component description, as a short text. - -- download_url: A direct URL to download the original file or archive documented - by this ABOUT file. - -- homepage_url: URL to the homepage for this component. - -- changelog_file: Changelog file for the component. - -- notes: Notes and comments about the component. - - -Optional Owner and Author fields -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- owner: The name of the primary organization or person(s) that owns or provides - the component. - -- owner_url: URL to the homepage for the owner. - -- contact: Contact information (such as an email address or physical address) - for the component owner. - -- author: Name of the organization(s) or person(s) that authored the component. - -- author_file: Author file for the component. - - -Optional Licensing fields -~~~~~~~~~~~~~~~~~~~~~~~~~ - -- copyright: Copyright statement for the component. - -- notice_file: Legal notice or credits for the component. - -- notice_url: URL to a legal notice for the component. - -- license_file: License file that applies to the component. For example, the - name of a license file such as LICENSE or COPYING file extracted from a - downloaded archive. - -- license_url: URL to the license text for the component. - -- license_expression: The DejaCode license expression that apply to the component. You - can separate each identifier using " or " and " and " to document the - relationship between multiple license identifiers, such as a choice among - multiple licenses. - -- license_name: The DejaCode license short name for the license. - -- license_key: The DejaCode license key(s) for the component. - - -Optional Boolean flag fields -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- redistribute: Set this flag to yes if the component license requires source - code redistribution. Defaults to no when absent. - -- attribute: Set this flag to yes if the component license requires publishing - an attribution or credit notice. Defaults to no when absent. - -- track_changes: Set this flag to yes if the component license requires tracking - changes made to a the component. Defaults to no when absent. - -- modified: Set this flag to yes if the component has been modified. Defaults to - no when absent. - -- internal_use_only: Set this flag to yes if the component is used internal only. - Defaults to no when absent. - -Optional Extension fields -~~~~~~~~~~~~~~~~~~~~~~~~~ - -You can create extension fields by prefixing them with a short prefix to -distinguish these from the standard fields (but this is not necessary). - - -Optional Extension fields to reference files stored in a version control system (VCS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These fields provide a simple way to reference files stored in a version control -system. There are many VCS tools such as CVS, Subversion, Git, ClearCase and GNU -Arch. Accurate addressing of a file or directory revision in each tool in a -uniform way may not be possible. Some tools may require access control via -user/password or certificate and this information should not be stored in an -ABOUT file. This extension defines the 'vcs' field extension prefix and a few -common fields to handle the diversity of ways that VCS tools reference files and -directories under version control: - -- vcs_tool: VCS tool such as git, svn, cvs, etc. - -- vcs_repository: Typically a URL or some other identifier used by a VCS tool to - point to a repository such as an SVN or Git repository URL. - -- vcs_path: Path used by a particular VCS tool to point to a file, directory or - module inside a repository. - -- vcs_tag: tag name or path used by a particular VCS tool. - -- vcs_branch: branch name or path used by a particular VCS tool. - -- vcs_revision: revision identifier such as a revision hash or version number. - - -Some examples for using the vcs_* extension fields include:: - - vcs_tool: svn - vcs_repository: http://svn.code.sf.net/p/inkscape/code/inkscape_project/ - vcs_path: trunk/inkscape_planet/ - vcs_revision: 22886 - -or:: - - vcs_tool: git - vcs_repository: git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git - vcs_path: tools/lib/traceevent - vcs_revision: b59958d90b3e75a3b66cd311661535f94f5be4d1 - - -Optional Extension fields for checksums -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These fields support checksums (such as SHA1 and MD5)commonly provided with -downloaded archives to verify their integrity. A tool can optionally use these -to verify the integrity of a file documented by an ABOUT file. - -- checksum_md5: MD5 for the file documented by this ABOUT file in the - "about_resource" field. - -- checksum_sha1: SHA1 for the file documented by this ABOUT file in the - "about_resource" field. - -- checksum_sha256: SHA256 for the file documented by this ABOUT file in the - "about_resource" field. - -Some examples:: - - checksum_md5: f30b9c173b1f19cf42ffa44f78e4b96c diff --git a/about b/about index 411f3279..1698fd7a 100755 --- a/about +++ b/about @@ -1,19 +1,23 @@ #!/bin/bash # -# Copyright (c) 2015 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # # cd to the root directory ABOUT_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$ABOUT_ROOT_DIR" -CONFIGURED_PYTHON=$ABOUT_ROOT_DIR/bin/python +# where we create a virtualenv +VIRTUALENV_DIR=venv + +CONFIGURED_PYTHON=$ABOUT_ROOT_DIR/$VIRTUALENV_DIR/bin/python if [ ! -f "$CONFIGURED_PYTHON" ]; then echo "* Configuring AboutCode ..." source $ABOUT_ROOT_DIR/configure fi -source $ABOUT_ROOT_DIR/bin/activate -$ABOUT_ROOT_DIR/bin/about "$@" +source $ABOUT_ROOT_DIR/$VIRTUALENV_DIR/bin/activate + +$ABOUT_ROOT_DIR/$VIRTUALENV_DIR/bin/about "$@" diff --git a/about.ABOUT b/about.ABOUT index e3493c37..4f778920 100644 --- a/about.ABOUT +++ b/about.ABOUT @@ -1,8 +1,8 @@ about_resource: . name: AboutCode-toolkit -version: 4.0.2 +version: 11.1.1 author: Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez -copyright: Copyright (c) 2013-2020 nexB Inc. +copyright: Copyright (c) nexB Inc. description: | AboutCode Toolkit is a tool to process ABOUT files. An ABOUT file provides a simple way to document the provenance (origin and license) @@ -15,6 +15,7 @@ licenses: key: apache-2.0 name: Apache License 2.0 url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 + spdx_license_key: Apache-2.0 notice_file: NOTICE owner: nexB Inc. vcs_repository: https://github.com/nexB/aboutcode-toolkit.git diff --git a/about.bat b/about.bat index 93c45b85..bf0a400f 100644 --- a/about.bat +++ b/about.bat @@ -1,5 +1,5 @@ @echo OFF -@rem Copyright (c) 2014 nexB Inc. http://www.nexb.com/ - All rights reserved. +@rem Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. @rem @@ -7,8 +7,11 @@ set ABOUT_ROOT_DIR=%~dp0 cd %ABOUT_ROOT_DIR% +# where we create a virtualenv +set VIRTUALENV_DIR=venv + set CMD_LINE_ARGS= -set CONFIGURED_PYTHON=%ABOUT_ROOT_DIR%\Scripts\python.exe +set CONFIGURED_PYTHON="%ABOUT_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts\python.exe" @rem Collect all command line arguments in a variable :collectarg @@ -25,11 +28,11 @@ goto about :configure echo * Configuring AboutCode ... - call %ABOUT_ROOT_DIR%\configure + call "%ABOUT_ROOT_DIR%\configure" :about -call %ABOUT_ROOT_DIR%\Scripts\activate -echo %ABOUT_ROOT_DIR%\bin\about %CMD_LINE_ARGS% -%ABOUT_ROOT_DIR%\bin\about %CMD_LINE_ARGS% +call "%ABOUT_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts\activate" +echo "%ABOUT_ROOT_DIR%\%VIRTUALENV_DIR%\bin\about" %CMD_LINE_ARGS% +"%ABOUT_ROOT_DIR%\%VIRTUALENV_DIR%\bin\about" %CMD_LINE_ARGS% :EOS diff --git a/apache-2.0.LICENSE b/apache-2.0.LICENSE index b1fac45f..29f81d81 100644 --- a/apache-2.0.LICENSE +++ b/apache-2.0.LICENSE @@ -1,4 +1,4 @@ -Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -198,4 +198,4 @@ Apache License distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file + limitations under the License. diff --git a/appveyor.yml b/appveyor.yml index ceb7ffed..f5fdd41d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,16 +1,16 @@ -version: '{build}' -install: - - configure etc/conf/dev +################################################################################ +# We use Appveyor to run minimal smoke tests suites on Pythons 3.x +# on Windows 64 bits +################################################################################ +environment: + matrix: + - PYTHON: "C:\\Python39-x64" -build: off -test_script: - - set - - py.test -vvs tests +build: off -on_success: - - "python etc/scripts/irc-notify.py aboutcode [{project_name}:{branch}] {short_commit}: \"{message}\" ({author}) {color_green}Succeeded,Details: {build_url},Commit: {commit_url}" -on_failure: - - "python etc/scripts/irc-notify.py aboutcode [{project_name}:{branch}] {short_commit}: \"{message}\" ({author}) {color_red}Failed,Details: {build_url},Commit: {commit_url}" +test_script: + - python -c "import sys;print(sys.getdefaultencoding())" + - cmd: "set PYTHON_EXECUTABLE=%PYTHON%\\python.exe && configure --dev && venv\\Scripts\\pytest -vvs tests" diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..475ea33b --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,70 @@ +################################################################################ +# We use Azure to run the full tests suites on multiple Python 3.x +# on multiple Windows, macOS and Linux versions all on 64 bits +# These jobs are using VMs with Azure-provided Python builds +################################################################################ + +jobs: + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ["3.9", "3.10", "3.11"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.13'] + test_suites: + all: make check + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs diff --git a/configure b/configure index 094a4682..6d317d4c 100755 --- a/configure +++ b/configure @@ -1,28 +1,201 @@ -#!/bin/bash +#!/usr/bin/env bash # -# Copyright (c) 2015 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. and others. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/ for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. # + +set -e +#set -x + +################################ +# A configuration script to set things up: +# create a virtualenv and install or update thirdparty packages. +# Source this script for initial configuration +# Use configure --help for details +# +# NOTE: please keep in sync with Windows script configure.bat +# +# This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz +# Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +################################ +CLI_ARGS=$1 + +################################ +# Defaults. Change these variables to customize this script +################################ + +# Requirement arguments passed to pip and used by default or with --dev. +REQUIREMENTS="--editable . --constraint requirements.txt" +DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" + +# where we create a virtualenv +VIRTUALENV_DIR=venv + +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" + +# extra arguments passed to pip +PIP_EXTRA_ARGS=" " + +# the URL to download virtualenv.pyz if needed +VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz ################################ -# change these variables to customize this script locally + + ################################ -# you can define one or more thirdparty dirs, each prefixed with TPP_DIR -export TPP_DIR="thirdparty" +# Current directory where this script lives +CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + -# default configurations -CONF_DEFAULT="etc/conf" -################################# +################################ +# Install with or without and index. With "--no-index" this is using only local wheels +# This is an offline mode with no index and no network operations +# NO_INDEX="--no-index " +NO_INDEX="" -CFG_CMD_LINE_ARGS="$@" -if [[ "$1" == "" ]]; then - # default for dev conf if not argument is provided - CFG_CMD_LINE_ARGS=$CONF_DEFAULT +################################ +# Thirdparty package locations and index handling +# Find packages from the local thirdparty directory if present +THIRDPARDIR=$CFG_ROOT_DIR/thirdparty +if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then + PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR" fi -CONFIGURE_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -if [[ "$PYTHON_EXE" == "" ]]; then - PYTHON_EXE=python +################################ +# Set the quiet flag to empty if not defined +if [[ "$CFG_QUIET" == "" ]]; then + CFG_QUIET=" " fi -$PYTHON_EXE "$CONFIGURE_ROOT_DIR/etc/configure.py" $CFG_CMD_LINE_ARGS + +################################ +# Find a proper Python to run +# Use environment variables or a file if available. +# Otherwise the latest Python by default. +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi + fi +} + + +################################ +create_virtualenv() { + # create a virtualenv for Python + # Note: we do not use the bundled Python 3 "venv" because its behavior and + # presence is not consistent across Linux distro and sometimes pip is not + # included either by default. The virtualenv.pyz app cures all these issues. + + VENV_DIR="$1" + if [ ! -f "$CFG_BIN_DIR/python" ]; then + + mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" + + if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then + VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" + else + VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" + wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" + fi + + $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ + --pip embed --setuptools embed \ + --seeder pip \ + --never-download \ + --no-periodic-update \ + --no-vcs-ignore \ + $CFG_QUIET \ + "$CFG_ROOT_DIR/$VENV_DIR" + fi +} + + +################################ +install_packages() { + # install requirements in virtualenv + # note: --no-build-isolation means that pip/wheel/setuptools will not + # be reinstalled a second time and reused from the virtualenv and this + # speeds up the installation. + # We always have the PEP517 build dependencies installed already. + + "$CFG_BIN_DIR/pip" install \ + --upgrade \ + --no-build-isolation \ + $CFG_QUIET \ + $PIP_EXTRA_ARGS \ + $1 +} + + +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + set +e + exit +} + + +################################ +# Main command line entry point +CFG_REQUIREMENTS=$REQUIREMENTS + +# We are using getopts to parse option arguments that start with "-" +while getopts :-: optchar; do + case "${optchar}" in + -) + case "${OPTARG}" in + help ) cli_help;; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; + esac;; + esac +done + + +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" + +find_python +create_virtualenv "$VIRTUALENV_DIR" +install_packages "$CFG_REQUIREMENTS" +. "$CFG_BIN_DIR/activate" + + +set +e diff --git a/configure.bat b/configure.bat index d1a9dc55..15ab7015 100644 --- a/configure.bat +++ b/configure.bat @@ -1,64 +1,203 @@ @echo OFF +@setlocal + +@rem Copyright (c) nexB Inc. and others. All rights reserved. +@rem SPDX-License-Identifier: Apache-2.0 +@rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +@rem See https://github.com/aboutcode-org/ for support or download. +@rem See https://aboutcode.org for more information about nexB OSS projects. + + +@rem ################################ +@rem # A configuration script to set things up: +@rem # create a virtualenv and install or update thirdparty packages. +@rem # Source this script for initial configuration +@rem # Use configure --help for details + +@rem # NOTE: please keep in sync with POSIX script configure + +@rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz +@rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default +@rem ################################ + + +@rem ################################ +@rem # Defaults. Change these variables to customize this script +@rem ################################ + +@rem # Requirement arguments passed to pip and used by default or with --dev. +set "REQUIREMENTS=--editable . --constraint requirements.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" + +@rem # where we create a virtualenv +set "VIRTUALENV_DIR=venv" + +@rem # Cleanable files and directories to delete with the --clean option +set "CLEANABLE=build dist venv .cache .eggs" + +@rem # extra arguments passed to pip +set "PIP_EXTRA_ARGS= " + +@rem # the URL to download virtualenv.pyz if needed +set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz +@rem ################################ + + +@rem ################################ +@rem # Current directory where this script lives +set CFG_ROOT_DIR=%~dp0 +set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + + +@rem ################################ +@rem # Thirdparty package locations and index handling +@rem # Find packages from the local thirdparty directory +if exist "%CFG_ROOT_DIR%\thirdparty" ( + set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" +) + + +@rem ################################ +@rem # Set the quiet flag to empty if not defined +if not defined CFG_QUIET ( + set "CFG_QUIET= " +) -@rem Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. @rem ################################ -@rem # change these variables to customize this script locally +@rem # Main command line entry point +set "CFG_REQUIREMENTS=%REQUIREMENTS%" + +:again +if not "%1" == "" ( + if "%1" EQU "--help" (goto cli_help) + if "%1" EQU "--clean" (goto clean) + if "%1" EQU "--dev" ( + set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" + ) + shift + goto again +) + +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" + + @rem ################################ -@rem # you can define one or more thirdparty dirs, each prefixed with TPP_DIR -set TPP_DIR=thirdparty - - -@rem # default configurations -set CONF_DEFAULT="etc/conf" -@rem ################################# - -set ABOUT_ROOT_DIR=%~dp0 -@rem !!!!!!!!!!! ATTENTION !!!!! -@rem there is a space at the end of the set SCANCODE_CLI_ARGS= line ... -@rem NEVER remove this! -@rem otherwise, this script and scancode do not work. - -set ABOUT_CLI_ARGS= -@rem Collect/Slurp all command line arguments in a variable -:collectarg - if ""%1""=="""" ( - goto continue - ) - call set ABOUT_CLI_ARGS=%ABOUT_CLI_ARGS% %1 - shift - goto collectarg - -:continue - -@rem default configuration when no args are passed -if "%ABOUT_CLI_ARGS%"==" " ( - set ABOUT_CLI_ARGS="%CONF_DEFAULT%" - goto configure +@rem # Find a proper Python to run +@rem # Use environment variables or a file if available. +@rem # Otherwise the latest Python by default. +if not defined PYTHON_EXECUTABLE ( + @rem # check for a file named PYTHON_EXECUTABLE + if exist "%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ( + set /p PYTHON_EXECUTABLE=<"%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" + ) else ( + set "PYTHON_EXECUTABLE=py" + ) ) -:configure -if not exist "c:\python27\python.exe" ( - echo( - echo On Windows, AboutCode requires Python 2.7.x 32 bits to be installed first. - echo( - echo Please download and install Python 2.7 ^(Windows x86 MSI installer^) version 2.7.10. - echo Install Python on the c: drive and use all default installer options. - echo Do NOT install Python v3 or any 64 bits edition. - echo Instead download Python from this url and see the README.rst file for more details: - echo( - echo https://www.python.org/ftp/python/2.7.15/python-2.7.15.msi - echo( - exit /b 1 + +@rem ################################ +:create_virtualenv +@rem # create a virtualenv for Python +@rem # Note: we do not use the bundled Python 3 "venv" because its behavior and +@rem # presence is not consistent across Linux distro and sometimes pip is not +@rem # included either by default. The virtualenv.pyz app cures all these issues. + +if not exist "%CFG_BIN_DIR%\python.exe" ( + if not exist "%CFG_BIN_DIR%" ( + mkdir "%CFG_BIN_DIR%" + ) + + if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ + --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" + ) else ( + if not exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ( + curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% + + if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% + ) + ) + %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ + --pip embed --setuptools embed ^ + --seeder pip ^ + --never-download ^ + --no-periodic-update ^ + --no-vcs-ignore ^ + %CFG_QUIET% ^ + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" + ) ) -call c:\python27\python.exe "%ABOUT_ROOT_DIR%etc\configure.py" %ABOUT_CLI_ARGS% -if %errorlevel% neq 0 ( - exit /b %errorlevel% +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% ) -if exist "%SCANCODE_ROOT_DIR%bin\activate" ( - "%SCANCODE_ROOT_DIR%bin\activate" + + +@rem ################################ +:install_packages +@rem # install requirements in virtualenv +@rem # note: --no-build-isolation means that pip/wheel/setuptools will not +@rem # be reinstalled a second time and reused from the virtualenv and this +@rem # speeds up the installation. +@rem # We always have the PEP517 build dependencies installed already. + +"%CFG_BIN_DIR%\pip" install ^ + --upgrade ^ + --no-build-isolation ^ + %CFG_QUIET% ^ + %PIP_EXTRA_ARGS% ^ + %CFG_REQUIREMENTS% + + +@rem ################################ +:create_bin_junction +@rem # Create junction to bin to have the same directory between linux and windows +if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( + rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ) -goto EOS +mklink /J "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" + +if %ERRORLEVEL% neq 0 ( + exit /b %ERRORLEVEL% +) + +exit /b 0 -:EOS + +@rem ################################ +:cli_help + echo An initial configuration script + echo " usage: configure [options]" + echo " " + echo The default is to configure for regular use. Use --dev for development. + echo " " + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo " " + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + exit /b 0 + + +@rem ################################ +:clean +@rem # Remove cleanable file and directories and files from the root dir. +echo "* Cleaning ..." +for %%F in (%CLEANABLE%) do ( + rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 + del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 +) +exit /b 0 diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst deleted file mode 100644 index 7f59d031..00000000 --- a/docs/CHANGELOG.rst +++ /dev/null @@ -1,261 +0,0 @@ -2020-05-05 - Release 4.0.2 - - * Upgrade license-expression library to v1.2 - * Fix the missing `multi_sort` filter for Jinja2 - * Update help text for `--vartext` - - -2019-10-17 - Release 4.0.1 - - * Declare to follow SemVer for versioning schema - * Update REFERENCE.rst and README.rst - * Update license-expression library - - -2019-10-09 - - Release 4.0.0 - - * Support filenames/path with special characters #310 #378 #392 - * Update ABOUT file format to match the specification - * Log version of which AbcTK was used #397 - * Fix the licenses (key, name, file) not in sync issue #406 - * Correct invalid msg for boolean fields #403 - * Remove the `about_file_path` key/column from input/output #364 - * Use ',' to support multiple files #404 - * Fix bugs in `transform` #408, #412 - -2018-11-15 - - Release 3.3.0 - - * Update the list of common license keys - * New UrlListField introduced for list of urls - * The UrlField is now only taking single URL value - * The owner is now a StringField instead of ListField - * Format the ordering of the generated ABOUT file (See https://github.com/nexB/aboutcode-toolkit/issues/349#issuecomment-438871444) - * '+' and '(' and ')' is now supported in license_expression - * The key 'about_resource_path' is removed - * Revert back the requirement of the 'name' field - * Add a new supported 'internal_use_only' key - -2018-10-23 - - Release 3.2.2 - - * Fix the version number - * `name` field is no longer a required field - -2018-10-22 - - Release 3.2.1 - - * The 'license' field is now become 'license_key' - * Multiple licenses support - * No support for referenceing multuiple resources - * Fix the incorrect boolean field behaviors - * Display number of errors/warnings in the error log - -2018-09-19 - - Release 3.1.3 - - * Minor update - -2018-09-19 - - Release 3.1.2 - - * New `--vartext` option for `attrib` - * Add support for `checksum_sha256` and `author_file` - * `check` command will not count INFO message as error when `--verbose` is set - * Update `track_change` to `track_changes` - * New `--filter` and `--mapping-output` options for `inventory` - -2018-6-25 - - Release 3.1.1 - - * No support of multiple occurrence keys in the input - * Updated the specification document - * Fixed bug that cause template processing error in attrib - - etc... - - -2018-6-8 Chin-Yeung Li - - Release 3.1.0 - - * Fixed JSON input from AboutCode manger export and ScanCode output - * Added a new option `mapping-file` to support using a custom file for mapping - * Change the name of the option `--show-all` to `--verbose` - * Better error handling for copying file with permission issue - * Support timestamp in attribution output - - etc... - - -2017-11-17 Chin-Yeung Li - - Release 3.0.* - - ABOUT files is now YAML formatted. - Supported license expression. - Supported JSON input and output format: https://github.com/nexB/aboutcode-toolkit/issues/246 and https://github.com/nexB/aboutcode-toolkit/issues/277 - Support Python 3: https://github.com/nexB/aboutcode-toolkit/issues/280 - Refined help texts - Refined USAGE texts - Refined SPECs - - Input key changes: - ================== - `about_file` is replaced by `about_file_path` - `dje_license_key` is replaced by `license_expression` - `version` is no longer a required field - `home_url` is now `homepage_url` - - API Updated: - ============ - - Break down the 3 major functions: `inventory`, `genabout` and `genattrib` into 3 subcommands: - i.e. - `about inventory`, `about generate` and `about attrib` - - - A new `check` subcommand: https://github.com/nexB/aboutcode-toolkit/issues/281 - - - Some options changes - `--extract_license` becomes `--fetch-license` - `--license_text_location` becomes `--license-notice-text-location` - - etc... - - -2016-06-27 Chin-Yeung Li - - Release 2.3.2 - - * Documentation updates - - -2016-03-29 Philippe Ombredanne - - Release 2.3.1 - - * Various minor bug fixes and improvements - * Support for the latest DejaCode API if you use DejaCode - - -2016-03-28 Philippe Ombredanne - - Release 2.3.0 - - * Various minor bug fixes and improvements - * Support for the latest DejaCode API if you use DejaCode - - -2015-10-23 Chin-Yeung Li - - Release 2.2.0 - - * Improved CLI error messages - * Fixed the filtering of dicts with empty values. - * Refined help texts - * Updated configure script - * Refactorings and code simplifications - * Fixed misleading error message when using invalid api_url - - -2015-10-09 Chin-Yeung Li - - Release 2.1.0 - - * Minor code refactoring - * Handle long path error on Windows OS when using genattrib with a zip - - -2015-09-29 Chin-Yeung Li - - Release 2.0.4 - - * Added support to run genattrib with a zip file and tests - * Display a "Completed" message once the generation is completed - - -2015-08-01 Chin-Yeung Li - - Release 2.0.3 - - * Fix the bug of using genattrib.py on Windows OS - * Display version when running about.py, genabout.py and genattrib.py - - -2015-07-07 Chin-Yeung Li - - Release 2.0.2 - - * Handle input's encoding issues - * Better error handling - * Writing to and reading from Windows OS with paths > 255 chars - - -2015-06-09 Chin-Yeung Li - - Release 2.0.1 - - * Configuration script fixes and updates basic documentation. - - -2015-05-05 Chin-Yeung Li - - Release 2.0.0 - - * Breaking API changes: - - * the dje_license field has been renamed to dje_license_key - * when a dje_license_key is present, a new dje_license_url will be - reported when fetching data from the DejaCode API. - * In genabout, the '--all_in_one' command line option has been removed. - It was not well specified and did not work as advertised. - - * in genattrib: - - * the Component List is now optional. - * there is a new experimental '--verification_location' command line - option. This option will be removed in the future version. Do not use - it. - * the '+' character is now supported in file names. - * several bugs have been fixed. - * error handling and error and warning reporting have been improved. - * New documentation in doc: UsingAboutCodetoDocumentYourSoftwareAssets.pdf - - -2014-11-05 Philippe Ombredanne - - Release 1.0.2 - - * Minor bug fixes and improved error reporting. - - -2014-11-03 Philippe Ombredanne - - Release 1.0.1 - - * Minor bug fixes, such as extraneous debug printouts. - - -2014-10-31 Philippe Ombredanne - - Release 1.0.0 - - * Some changes in the spec, such as supporting only text in external - files. - * Several refinements including support for common licenses. - -2014-06-24 Chin-Yeung Li - - Release 0.8.1 - - * Initial release with minimal capabilities to read and validate - ABOUT files format 0.8.0 and output a CSV inventory. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..94f686b2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,28 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild +SOURCEDIR = source +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md deleted file mode 100644 index 7632e77f..00000000 --- a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.md +++ /dev/null @@ -1,470 +0,0 @@ -#**Using AboutCode Toolkit to Document Your Software Assets** - -[Using AboutCode Toolkit to Document Your Software Assets](#UsingAboutCodeToolkittoDocumentYourSoftwareAssets) -[AboutCode Toolkit Defined](#AboutCodeToolkitDefined) -[Key Terminology](#KeyTerminology) -[Using gen to Generate AboutCode Toolkit Files](#UsinggentoGenerateAboutCodeToolkitFiles) -* [Prepare Your Software Inventory for gen Standard Column Names](#PrepareYourSoftwareInventoryforgenStandardColumnNames) -* [Fields Renaming and Optional Custom Fields](#FieldsRenamingOptionalCustomFields) -* [Run gen to Generate AboutCode Toolkit Files](#RungentoGenerateAboutCodeToolkitFiles) -[Using attrib to Generate a Product Attribution Notice Package](#UsingattribtoGenerateaProductAttributionNoticePackage) -* [Prepare a Filtered Product BOM to Use as Input to attrib](#PrepareaFilteredProductBOMtoUseasInputtoattrib) -* [Prepare an Attribution Template to Use as Input to attrib](#PrepareanAttributionTemplatetoUseasInputtoattrib) -[Use jinja2 Features to Customize Your Attribution Template](#Usejinja2FeaturestoCustomizeYourAttributionTemplate) -* [Run attrib to Generate a Product Attribution Notice Package](#RunattribtoGenerateaProductAttributionNoticePackage) -[Using inventory to Generate a Software Inventory](#UsinginventorytoGenerateaSoftwareInventory) -* [Generate a Software Inventory of Your Codebase from AboutCode Toolkit Files](#GenerateaSoftwareInventory) - -#AboutCode Toolkit Defined - -AboutCode Toolkit is a tool for your software development team to document your code inside your codebase, typically in preparation for a product release, side-by-side with the actual code. AboutCode Toolkit files have a simple, standard format that identifies components and their associated licenses. The current AboutCode Toolkit subcommands are: - -* **attrib**: Generate a Product Attribution notice document (HTML format) from your AboutCode Toolkit files. You can also generate documents for other purposes (such as a License Reference) by varying your input control file and your .html template. - -* **check**: A simple command to validate the AboutCode Toolkit files and output errors/warnings if any on the terminal. - -* **gen**: Create AboutCode Toolkit files from a Software Inventory file (.csv or .json format) which is typically created from a software audit, and insert these AboutCode Toolkit files into your codebase. You can regenerate the AboutCode Toolkit files from a new Software Inventory file whenever you make changes. - -* **inventory**: Generate a Software Inventory list (.csv or .json format) from your codebase based on your AboutCode Toolkit files. Note that this Software Inventory will only include components that have AboutCode Toolkit data. In another word, if you do not create AboutCode Toolkit files for your own original software components, these components will not show up in the generated inventory. - -* **transform**: A command to transform an input CSV by applying renaming and filters and then output to a new CSV. - -Additional AboutCode Toolkit information is available at: - -* [http://www.aboutcode.org/](http://www.aboutcode.org/) for an overview and a link to the ABOUT File specification. - -* [https://github.com/nexB/aboutcode-toolkit/](https://github.com/nexB/aboutcode-toolkit/) for the AboutCode Toolkit tools. - -# Key Terminology - -Some key terminology that applies to AboutCode Toolkit tool usage: - -* **Software Inventory or Inventory** - means a list of all of the components in a Development codebase and the associated data about those components with a focus on software pedigree/provenance- related data for open source and third-party components. - -* **Product BOM or BOM** - means a subset list of the components in a Development codebase (Software Inventory) that are Deployed on a particular Product Release (a Product Bill of Materials). - -# Using gen to Generate AboutCode Toolkit Files - -## Prepare Your Software Inventory for gen Standard Column Names - -You should start with a software inventory of your codebase in spreadsheet format. You need to prepare a version of it that will identify the column values that you want to appear in your .ABOUT files. Note the following standard column names (defined in the ABOUT File Specification), which gen will use to look for the values that it will store in your generated .ABOUT files, as well as any additional text files that you identify, which it will copy and store next to the .ABOUT files. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Standard Column NameDescriptionNotes
about_resourceName/path of the component resourceMandatory.
nameComponent nameMandatory
versionComponent versionOptional
download_urlDirect URL to download the original file or archive documented by this ABOUT fileOptional
descriptionComponent descriptionOptional
homepage_urlURL to the homepage for this componentOptional
notesnotes textOptional
license_expressionExpression for the license of the component using DejaCode Enterprise license key(s).Optional. You can separate each identifier using " or " and " and " to document the relationship between multiple license identifiers, such as a choice among multiple licenses.
license_keyDejaCode Enterprise license key for the component.Optional. gen will obtain license information from DejaCode Enterprise if the --fetch-license option is set, including the license text, in order to create and write the appropriate .LICENSE file in the .ABOUT file target directory.
license_nameLicense name for the component.Optional. This field will be generated if the --fetch-license option is set.
license filelicense file nameOptional. gen will look for the file name (if a directory is specified in the --reference option) to copy that file to the .ABOUT file target directory.
license_urlURL to the license text for the componentOptional
copyrightcopyright statement for the componentOptional
notice_filenotice text file nameOptional
notice_urlURL to the notice text for the componentOptional
redistributeYes/No. Does the component license require source redistribution.Optional
attributeYes/No. Does the component license require publishing an attribution or credit notice.Optional
track_changesYes/No. Does the component license require tracking changes made to the component.Optional
modifiedYes/No. Have the component been modified.Optional
internal_use_onlyYes/No. Is the component internal use only.Optional
changelog_filechangelog text file nameOptional
ownername of the organization or person that owns or provides the componentOptional
owner_urlURL to the owner for the componentOptional
contactContact informationOptional
authorauthor of the componentOptional
author_fileauthor text file nameOptional
vcs_toolName of the version control tool.Optional
vcs_repositoryName of the version control repository.Optional
vcs_pathName of the version control path.Optional
vcs_tagName of the version control tag.Optional
vcs_branchName of the version control branch.Optional
vcs_revisionName of the version control revision.Optional
checksum_md5MD5 value for the fileOptional
checksum_sha1SHA1 value for the fileOptional
checksum_sha256SHA256 value for the fileOptional
spec_versionThe version of the ABOUT file format specification used for this file. Optional
- - -## Fields Renaming and Optional Custom Fields - -Since your input's column name may not match with the AboutCode Toolkit standard field name, you can use the transform subcommand to do the transformation. - -A transform configuration file is used to describe which transformations and validations to apply to a source CSV file. This is a simple text file using YAML format, using the same format as an .ABOUT file. - -The attributes that can be set in a configuration file are: - -* column_renamings: -An optional map of source CSV column name to target CSV new column name that -is used to rename CSV columns. - -For instance with this configuration the columns "Directory/Location" will be -renamed to "about_resource" and "foo" to "bar": - - column_renamings: - 'Directory/Location' : about_resource - foo : bar - -The renaming is always applied first before other transforms and checks. All -other column names referenced below are these that exist AFTER the renaming -have been applied to the existing column names. - -* required_columns: -An optional list of required column names that must have a value, beyond the -standard columns names. If a source CSV does not have such a column or a row is -missing a value for a required column, an error is reported. - -For instance with this configuration an error will be reported if the columns -"name" and "version" are missing or if any row does not have a value set for -these columns: - - required_columns: - - name - - version - -* column_filters: -An optional list of column names that should be kept in the transformed CSV. If -this list is provided, all the columns from the source CSV that should be kept -in the target CSV must be listed be even if they are standard or required -columns. If this list is not provided, all source CSV columns are kept in the -transformed target CSV. - -For instance with this configuration the target CSV will only contains the "name" -and "version" columns and no other column: - - column_filters: - - name - - version - - -## Run gen to Generate AboutCode Toolkit Files - -When your software inventory is ready, you can save it as a .csv file, and use it as input to run gen to generate your AboutCode Toolkit files. The official gen parameters are defined here: - -* [https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst](https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst) - -Here is an example of a gen command: - -about gen --fetch-license {{your license library api}} {{your license library api key}} --reference /Users/harrypotter/myAboutFiles/ /Users/harrypotter/myAboutFiles/myProject-bom.csv /Users/harrypotter/myAboutFiles/ - -Note that this example gen command does the following: - -* Activates the --fetch-license option to get license text. - -* Activates the --reference option to get license and notice text files that you have specified in your software inventory to be copied next to the associated .ABOUT files when those are created. - -* Specifies the path of the software inventory to control the processing. - -* Specifies a target output directory. - -Review your generated AboutCode Toolkit files to determine if they meet your requirements. Here is a simple example of a linux-redhat-7.2.ABOUT file that documents the directory /linux-redhat-7.2/ : - - about_resource: . - name: Linux RedHat - version: v 7.2 - attribute: Y - copyright: Copyright (c) RedHat, Inc. - license_expression: gpl-2.0 - licenses: - - key: gpl-2.0 - name: GNU General Public License 2.0 - file: gpl-2.0.LICENSE - owner: Red Hat - redistribute: Y - -You can make the appropriate changes to your input software inventory and then run gen as often as necessary to replace the generated AboutCode Toolkit files with the improved output. - -# Using attrib to Generate a Product Attribution Notice Package - -## Prepare a Filtered Product BOM to Use as Input to attrib - -The Software Inventory that you prepared for gen most likely includes components that do not need to appear in a product attribution notice package; for example: - -* Components in your codebase that are not Deployed on the final product (e.g. build tools, testing tools, internal documentation). - -* Components in your codebase under licenses that do not require attribution (e.g. proprietary packages, commercial products). - -There are two options here: - -* Edit the jinja2 template to only include the one that your want to include in the attrib such as: - - {% if about_object.attribute.value %} - ... - -* You should prepare a filtered version of your software inventory (the one that you used for gen) by removing the rows that identify components which should not be included in a product attribution notice package, and saving that filtered version as your Product BOM. - -You should also order the rows in this Product BOM in the sequence that you would like them to appear in the product attribution notice package. - -## Prepare an Attribution Template to Use as Input to attrib - -You can run attrib using the default_html.template (or default_json.template if want JSON output) provided with the AboutCode Toolkit tools: - -[https://github.com/nexB/aboutcode-toolkit/blob/develop/templates/default_html.template](https://github.com/nexB/aboutcode-toolkit/blob/develop/templates/default_html.template) - -If you choose to do that, you will most likely want to edit the generated .html file to provide header information about your own organization and product. - -Running attrib with the default_html.template file is probably your best choice when you are still testing your AboutCode Toolkit process. Once you have a good understanding of the generated output, you can customize the template to provide the standard text that you want to see whenever you generate product attribution for your organization. You can also create alternative versions of the template to use attrib to generate other kinds of documents, such as a License Reference. - -### Use jinja2 Features to Customize Your Attribution Template - -The attrib tool makes use of the open source python library **jinja2** ([http://jinja.pocoo.org/docs/dev/templates/](http://jinja.pocoo.org/docs/dev/templates/)) in order to extend .html capabilities and transform AboutCode Toolkit input data into the final format of the generated attribution file. The **default.html **file contains text that complies with jinja2 syntax specifications in order to support grouping, ordering, formatting and presentation of your AboutCode Toolkit data. If your attribution requirements are complex, you may wish to study the jinja2 documentation to modify the default.html logic; alternatively, here are a few relatively simple concepts that relate to the attribution document domain. - -The simplest modifications to the default_html.template file involve the labels and standard text. For example, here is the default template text for the Table of Contents: - -
- {% for about_object in abouts %} -

{{ about_object.name.value }} - {% if about_object.version.value %} {{ about_object.version.value }} - {% endif %}

- {% endfor %} -
- -If you would prefer something other than a simple space between the component name and the component version, you can modify it to something like this: - -
- {% for about_object in abouts %} -

{{ about_object.name.value }} - {% if about_object.version.value %} - Version {{ about_object.version.value }} - {% endif %}

- {% endfor %} -
- -The "if about_object.version.value" is checking for a component version, and if one exists it generates output text that is either a space followed by the actual version value, or, as in this customized template, it generates output text as " - Version ", followed by the actual version value. You will, of course, want to test your output to get exactly the results that you need. - -Note that you can actually use attrib to generate an AboutCode Toolkit-sourced document of any kind for varying business purposes, and you may want to change the grouping/ordering of the data for different reporting purposes. (Here we get into somewhat more complex usage of jinja2 features, and you may wish to consult the jinja2 documentation to reach a more comprehensive understanding of the syntax and features.) The default ordering is by component, but In the following example, which is intended to support a "license reference" rather than an attribution document, the customized template modifies the data grouping to use a custom field called "confirmed license": - -
- {% for group in abouts | groupby('confirmed_license') %} -

- {% for license in group.grouper.value %} - {{ license }} - - {% endfor %} -

- {% endfor %} -
- -After the table of contents, this example customized template continues with the license details using the jinja2 for-loop capabilities. Notice that the variable "group.grouper.value" is actually the license name here, and that “License URL” can be any URL that you have chosen to store in your .ABOUT files: - - {% for group in abouts | groupby('confirmed_license') %} - {% for confirmed_license in group.grouper.value %} - -
-

{{ confirmed_license }}

-

This product contains the following open source software packages licensed under the terms of the license: {{confirmed_license}}

- -
- {%for about_object in group.list %} - {% if loop.first %} - {% if about_object.license_url.value %} - {% for lic_url in about_object.license_url.value %} -

License URL: {{lic_url }}

- {% endfor %} - {% endif %} - {% endif %} -
  • - {{ about_object.name.value }}{% if about_object.version.value %} - Version - {{ about_object.version.value }}{% endif %} -
    • - {% if about_object.copyright.value %}
    {{about_object.copyright.value}}
    {% endif %} - {% if loop.last %} - 
    -                    {% for lic_key in about_object.license_file.value %}
-                        {{about_object.license_file.value[lic_key]}}
-                    {% endfor %}
-
    - {% endif %} - {% endfor %} -
    -
    -
    - {% endfor %} - {% endfor %} -
    - - -In summary, you can start with simple, cosmetic customizations to the default.html template, and gradually introduce a more complex structure to the attrib output to meet varying business requirements. - -## Run attrib to Generate a Product Attribution Notice Package - -When your Product BOM (your filtered software inventory) is ready, you can save it as a .csv file, and use it as input to run attrib to generate your product attribution notice package. The official attrib parameters are defined here: - -* [https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst](https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst) - -Here is an example of a attrib command: - -about attrib --template /Users/harrypotter/myAboutFiles/my_attribution_template_v1.html /Users/harrypotter/myAboutFiles/ /Users/harrypotter/myAboutFiles/myProject-attribution-document.html - -Note that this example attrib command does the following: - -* Activates the --template option to specify a custom output template. - -* Specifies the path of the AboutCode Toolkit files needed to generate the output document. - -* Specifies the full path (include file name) of the output document to be generated. - -A successful execution of attrib will create a .html (or .json depends on the template) file that is ready to use to meet your attribution requirements. - -# Using inventory to Generate a Software Inventory - -## Generate a Software Inventory of Your Codebase from AboutCode Toolkit Files - -One of the major features of the ABOUT File specification is that the .ABOUT files are very simple text files that can be created, viewed and edited using any standard text editor. Your software development and maintenance processes may require or encourage your software developers to maintain .ABOUT files and/or associated text files manually. For example, when a developer addresses a software licensing issue with a component, it is appropriate to adjust the associated AboutCode Toolkit files manually. - -If your organization adopts the practice of manually creating and maintaining AboutCode Toolkit files, you can easily re-create your software inventory from your codebase using inventory. The official inventory parameters are defined here: - -* [https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst](https://github.com/nexB/aboutcode-toolkit/blob/develop/REFERENCE.rst) - -A successful execution of inventory will create a complete software inventory in .csv format or .json format based on defined format. diff --git a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.pdf b/docs/UsingAboutCodetoDocumentYourSoftwareAssets.pdf deleted file mode 100644 index 4b371b36..00000000 Binary files a/docs/UsingAboutCodetoDocumentYourSoftwareAssets.pdf and /dev/null differ diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..4a3c1a48 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,47 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +if "%SPHINXAUTOBUILD%" == "" ( + set SPHINXAUTOBUILD=sphinx-autobuild +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +if "%1" == "docs" goto docs + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:docs +@echo +@echo Starting up the docs server... +@echo +%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh new file mode 100644 index 00000000..922097b1 --- /dev/null +++ b/docs/scripts/doc8_style_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Check for Style Code Violations +doc8 --max-line-length 100 source --ignore D000 --quiet diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh new file mode 100644 index 00000000..3b271568 --- /dev/null +++ b/docs/scripts/sphinx_build_link_check.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# halt script on error +set -e +# Build locally, and then check links +sphinx-build -E -W -b linkcheck source build diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js new file mode 100644 index 00000000..0da0cb8d --- /dev/null +++ b/docs/source/_static/js/custom.js @@ -0,0 +1,4 @@ +$(document).ready(function () { + $('a[href^="file://"], a[href^="http://"], a[href^="https://"]').not('a[class*=internal]').attr('target', '_blank'); + $('a[href$=".docx"], a[href$=".xlsx"]').not('a[class*=internal]').attr('target', '_self'); + }); diff --git a/docs/source/_static/theme_overrides-skeleton-2022-03-28-updated.css b/docs/source/_static/theme_overrides-skeleton-2022-03-28-updated.css new file mode 100644 index 00000000..50e83d9b --- /dev/null +++ b/docs/source/_static/theme_overrides-skeleton-2022-03-28-updated.css @@ -0,0 +1,2142 @@ +body { + color: #000000; +} + +html { + font-size: 14px; +} + +p { + line-height: 24px; + line-height: 20px; + font-size: 16px; + font-size: 14px; + margin: 0 0 24px; + margin-top: 0px; + margin-bottom: 24px; + margin-bottom: 0px; + margin-bottom: 10px; +} + +.wy-plain-list-disc, +.rst-content .section ul, +.rst-content .toctree-wrapper ul, +article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} + +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} + +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} + +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} + +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + + + + + +/* =============================== */ + +/* custom-alert-01 */ +.custom-alert-01 .admonition-title { + color: #000000; + background: #00e673; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #00e673; + padding: 10px 10px 7px 10px; +} + +.custom-alert-01 .admonition-title::before { + content: ""; +} + +div.custom-alert-01.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #00e673; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #b8b8b8; + box-shadow: none; + margin: 0px 0px 20px 0px; +} + +div.custom-alert-01.admonition p { + font-size: 14px; + line-height: 15px; + line-height: 20px; +} + +div.custom-alert-01.admonition p.admonition-title { + font-size: 16px; + font-size: 14px; + color: #000000; + line-height: 10px; +} + + +/* custom-alert-02 */ +.custom-alert-02 .admonition-title { + color: #000000; + color: #ffffff; + background: #00e673; + background: #008888; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #00e673; + border-bottom: solid 1px #008888; + padding: 10px 10px 7px 10px; +} + +.custom-alert-02 .admonition-title::before { + content: ""; +} + +div.custom-alert-02.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #00e673; + border: solid 1px #008888; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #b8b8b8; + box-shadow: none; + margin: 0px 0px 20px 0px; +} + +div.custom-alert-02.admonition p { + font-size: 14px; + line-height: 15px; + line-height: 20px; +} + +div.custom-alert-02.admonition p.admonition-title { + font-size: 16px; + font-size: 14px; + color: #000000; + color: #ffffff; + line-height: 10px; +} + +/* ==================================================== */ + +/* warning */ +.rst-content .warning .admonition-title { + color: #000000; + background: #ffcccc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #ff8888; +} + +.rst-content .warning { + color: #000000; + background: #ffffff; + border: solid 1px #ff8888; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #cccccc; + margin: 20px 0px 30px 0px; +} + + +/* caution */ +.rst-content .caution .admonition-title { + color: #000000; + background: #ffff66; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} + +.rst-content .caution { + color: #000000; + background: #ffffff; + border: solid 1px #ffff33; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #cccccc; + margin: 20px 0px 30px 0px; +} + + +/* note */ +.rst-content .note .admonition-title { + color: #000000; + background: #ccffff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #00ccff; +} + +.rst-content .note { + color: #000000; + background: #ffffff; + border: solid 1px #00ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #cccccc; + margin: 20px 0px 30px 0px; +} + + +/* note01 */ +.rst-content .note01 .admonition-title { + color: #000000; + background: #e8e8e8; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} + +.rst-content .note01 { + color: #000000; + background: #ffffff; + border: solid 1px #000000; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #cccccc; + margin: 20px 0px 30px 0px; +} + + +/* note02 */ +.rst-content .note02 .admonition-title { + color: #000000; + background: #ffffff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} + +.rst-content .note02 { + color: #000000; + background: #ffffff; + border: solid 1px #000000; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #cccccc; + margin: 20px 0px 30px 0px; +} + +/* ==================================================== */ + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", Courier, monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066ff; + text-decoration: none; +} + +.wy-nav-content-wrap a:hover { + color: #0066ff; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} + +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} + +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} + +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + background-color: transparent; + border: 0; + font-style: normal; + font-family: Inconsolata, Consolas, Monaco, "Lucida Console", monospace; + font-size: 13px; + font-weight: 500; + padding: 1px 3px 1px 3px; +} + +.rst-content pre.literal-block, +.rst-content div[class^="highlight"] pre, +.rst-content .linenodiv pre { + font-family: Inconsolata, Consolas, Monaco, "Lucida Console", monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + line-height: 1.4; + color: #404040; +} + +.rst-content pre.literal-block, +.rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, +.rst-content tt, +.rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 2px; + font-size: 12px; + background-color: #e8e8e8; +} + +.rst-content code, +.rst-content tt, +code { + font-family: Monaco, Menlo, Consolas, Courier New, monospace; + font-size: 12px; + border: solid 1px #d8d8d8; + border-color: #e8e8e8; + background-color: #f9f9f9; + padding: 1px 2px 1px 2px; + font-weight: 500; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #cd0000; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +.rst-content blockquote { + margin-left: 2px; + margin-top: 20px !important; + margin-bottom: 20px !important; + border-left: solid 2px #e8e8e8; + padding-left: 20px; + line-height: 24px; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, +.rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, +.rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, +.rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, +.rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, +article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + margin-bottom: 0px; +} + +/* add top margin so image source below image has some space between text and image above */ +.float-left { + float: left; + margin-right: 20px; + margin-top: 10px; +} + +/* and for balance: */ +.float-right { + float: right; + margin-top: 10px; +} + +/* for the nav buttons in breadcrumbs */ +.rst-breadcrumbs-buttons .float-left, +.rst-breadcrumbs-buttons .float-right { + float: right; + float: left; + margin-top: 10px; + margin-top: 0px; + margin-top: 10px; + margin-bottom: 10px; + margin-bottom: 0px; +} + +img { + border: solid 1px #cccccc; + box-shadow: none; + margin-top: 20px; + margin-bottom: 0px; +} + +.forkme_img { + margin-top: 0px; +} + +/* ===== */ +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} + +/* ===== */ + +/* 1/13/2022 Thursday 11:32:56 AM. Handles the navigation tree on the left. */ +.wy-menu-vertical li.toctree-l1.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l2.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l3.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l4.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l5.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l6.current>a.reference.internal.current, +.wy-menu-vertical li.toctree-l7.current>a.reference.internal.current { + background-color: #008888; + color: #ffffff; + font-weight: 600; + padding-top: 8px; + border-right: 0; + font-size: 13px; + + border: 0; + font-weight: 500; + background-color: #0066ff; + background-color: #008888; +} + +.wy-menu-vertical li.toctree-l1.current>a.reference.internal.current:hover, +.wy-menu-vertical li.toctree-l2.current>a.reference.internal.current:hover, +.wy-menu-vertical li.toctree-l3.current>a.reference.internal.current:hover, +.wy-menu-vertical li.toctree-l4.current>a.reference.internal.current:hover, +.wy-menu-vertical li.toctree-l5.current>a.reference.internal.current:hover { + background-color: #008888; + color: #ffffff; + text-decoration: none; + font-weight: 600; + border-right: 0; + cursor: default; + + font-weight: 500; + background-color: #0066ff; + background-color: #008888; +} + +.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a, +.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a, +.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a, +.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a, +.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a, +.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a, +.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a, +.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a, +.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a { + display: block; + color: #0066ff; + color: #0033cc; + font-size: 13px; + border-right: 0; + + padding-top: 8px; +} + +.wy-menu-vertical li.toctree-l2.current>a, +.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a, +.wy-menu-vertical li.toctree-l3.current>a, +.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a { + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; + color: #0066ff; + color: #0033cc; + + padding-top: 8px; + border-top: solid 1px transparent; + border-bottom: solid 1px transparent; +} + +.wy-menu-vertical li.toctree-l1.current>a { + border-top: 0; + border-bottom: 0; + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; + color: #0066ff; + color: #0033cc; +} + +/* non-selected node inside expanded TOC section */ +.wy-menu-vertical li.toctree-l2 a, +.wy-menu-vertical li.toctree-l3 a, +.wy-menu-vertical li.toctree-l4 a, +.wy-menu-vertical li.toctree-l5 a, +.wy-menu-vertical li.toctree-l6 a, +.wy-menu-vertical li.toctree-l7 a, +.wy-menu-vertical li.toctree-l8 a, +.wy-menu-vertical li.toctree-l9 a, +.wy-menu-vertical li.toctree-l10 a { + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; + color: #0066ff; + color: #0033cc; + border-right: 0; +} + +.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover, + +.wy-menu-vertical li.toctree-l1>a:hover, +.wy-menu-vertical li.toctree-l2>a:hover, +.wy-menu-vertical li.toctree-l3>a:hover, +.wy-menu-vertical li.toctree-l4>a:hover, +.wy-menu-vertical li.toctree-l5>a:hover { + background-color: #000000; + color: #ffffff; +} + +.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a:hover, +.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a:hover, +.wy-menu-vertical li.toctree-l4.current li.toctree-l5>a:hover, +.wy-menu-vertical li.toctree-l5.current li.toctree-l6>a:hover, +.wy-menu-vertical li.toctree-l6.current li.toctree-l7>a:hover, +.wy-menu-vertical li.toctree-l7.current li.toctree-l8>a:hover, +.wy-menu-vertical li.toctree-l8.current li.toctree-l9>a:hover, +.wy-menu-vertical li.toctree-l9.current li.toctree-l10>a:hover, +.wy-menu-vertical li.toctree-l10.current li.toctree-l11>a:hover { + display: block; + background-color: #000000; + color: #ffffff; + font-size: 13px; + border-right: 0; +} + +.wy-menu-vertical li.toctree-l1 a button.toctree-expand, +.wy-menu-vertical li.toctree-l2 a button.toctree-expand, +.wy-menu-vertical li.toctree-l3 a button.toctree-expand { + color: #cccccc; +} + +.wy-menu-vertical li.toctree-l1 a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand { + color: #ffffff; +} + +.wy-side-nav-search { + display: block; + width: 300px; + padding: .809em; + padding-bottom: 20px; + margin-bottom: .809em; + z-index: 200; + background-color: #202020; + background-color: #2980b9; + border-bottom: solid 1px #666666; + border-bottom: solid 1px #000000; + text-align: center; + color: #d8d8d8; + color: #ffffff; +} + +.wy-side-nav-search a { + color: #a8a8a8; + color: #ffffff; + font-weight: 500; +} + +.wy-side-nav-search a:hover { + background-color: transparent; + color: #a8a8a8; + color: #ffffff; + text-decoration: underline; +} + +.wy-side-nav-search>div.version { + margin-top: -.4045em; + margin-bottom: .809em; + font-weight: 400; + color: #cccccc; +} + +.wy-side-nav-search input[type="text"] { + border-color: #d8d8d8; + background-color: #ffffff; + box-shadow: none; +} + +/* From the ansible RTD, which has a nice top navbar */ + +/*! Override sphinx rtd theme max-with of 800px */ +.wy-nav-content { + max-width: 100% +} + +/*! Override sphinx_rtd_theme - keeps left-nav from overwriting Documentation title */ +.wy-nav-side { + top: 44px; + padding-bottom: 45px; + border-right: solid 1px #e8e8e8; + border-right-color: #e8e8e8; + background-color: #202020; + background-color: #343131; +} + +/* What does this do? */ +.DocSite-nav { + display: none +} + +.ansibleNav { + background: #ffffff; + background-color: #ffffff; + padding: 0 20px; + width: auto; + border-bottom: 4px solid #000000; + border-top: solid 1px #0066ff; + border-top: none; + font-size: 14px; + z-index: 300; + box-shadow: 1px 1px 1px 1px #333333; + box-shadow: none; +} + +.ansibleNav ul { + list-style: none; + padding-left: 0; + margin-top: 0; + padding-top: 0px; +} + +.ansibleNav ul li { + padding: 7px 0; + border-bottom: 1px solid #444; +} + +.ansibleNav ul li:last-child { + border: none +} + +.ansibleNav ul li a { + color: #ffffff; + text-decoration: none; + padding: 6px 0; +} + +.ansibleNav ul li a:hover { + color: #66ffff; + background: 0 0; +} + +@media screen and (min-width:768px) { + .DocSite-globalNav { + display: block; + position: fixed + } + + #sideBanner { + display: block + } + + .DocSite-sideNav { + display: none + } + + .DocSite-nav { + flex: initial; + -webkit-flex: initial; + display: flex; + display: -webkit-flex; + flex-direction: row; + -webkit-flex-direction: row; + justify-content: flex-start; + -webkit-justify-content: flex-start; + padding: 15px; + background-color: #000; + text-decoration: none; + font-family: "Open Sans", sans-serif + } + + .DocSiteNav-logo { + width: 28px; + height: 28px; + margin-right: 8px; + margin-top: -6px; + position: fixed; + z-index: 1 + } + + .DocSiteNav-title { + color: #fff; + font-size: 20px; + position: fixed; + margin-left: 40px; + margin-top: -4px; + z-index: 1 + } + + .ansibleNav { + height: 45px; + width: 100%; + font-size: 13px; + padding: 0 60px 0 0 + } + + .ansibleNav ul { + float: right; + display: flex; + flex-wrap: nowrap; + margin-top: 13px + } + + .ansibleNav ul li { + padding: 0; + border-bottom: none + } + + .ansibleNav ul li a { + color: #0066ff; + text-decoration: none; + padding: 8px 13px; + } +} + +@media screen and (min-width:768px) { + + #sideBanner, + .DocSite-globalNav { + display: block + } + + .DocSite-sideNav { + display: none + } + + .DocSite-nav { + flex: initial; + -webkit-flex: initial; + display: flex; + display: -webkit-flex; + flex-direction: row; + -webkit-flex-direction: row; + justify-content: flex-start; + -webkit-justify-content: flex-start; + padding: 15px; + background-color: #000; + text-decoration: none; + font-family: "Open Sans", sans-serif + } + + .DocSiteNav-logo { + width: 28px; + height: 28px; + margin-right: 8px; + margin-top: -6px; + position: fixed + } + + .DocSiteNav-title { + color: #0066ff; + font-size: 22px; + font-family: Arial; + position: fixed; + /* margin-left:70px; */ + margin-top: -7px; + z-index: 300; + background-color: transparent; + padding: 0px 5px 2px 5px; + border: 0; + } + + .DocSiteNav-title:hover { + text-decoration: underline; + color: #000000; + } + + .ansibleNav { + height: 45px; + font-size: 13px; + padding: 0 150px 0 0; + } + + .ansibleNav ul { + float: right; + display: flex; + flex-wrap: nowrap; + margin-top: 0px; + } + + .ansibleNav ul li { + padding: 0; + border-bottom: none; + } + + /* The dropdown content links color is controlled by: +.dropdown:hover .dropdown-content-button a, +.dropdown:hover .dropdown-content a { +*/ + + .ansibleNav ul li a { + color: #ffffff; + text-decoration: none; + padding: 3px 10px; + padding: 6px 10px; + } + + /* .ansibleNav .turnip ul li a { */ + .ansibleNav ul li .turnip a { + padding: 3px 10px 2px 20px; + margin-top: -3px; + } + + .ansibleNav ul li .turnip a::before { + content: "• "; + color: #212529; + } + +} + +.dropbtn { + display: inline-block; + color: #0066ff; + background-color: transparent; + text-align: center; + padding: 13px 15px 7px 15px; + text-decoration: none; + vertical-align: top; + border: 0; + font-family: "Lato"; + font-size: 17px; +} + +.dropdown:hover .dropbtn { + background-color: #0066ff; + background-color: #008888; + color: #ffffff; + border-bottom: solid 4px #0066ff; + border-bottom-color: #000000; +} + +.dropdown:hover .dropbtn:hover { + background-color: #0066ff; + background-color: #008888; +} + +/* Commenting out these 2 does NOT stop the on-hover dropdown behavior. */ +li.dropdown { + display: inline-block; +} + +li.dropdown:hover { + display: inline-block; +} + +.dropdown-content { + display: none; + position: absolute; + background-color: #000000; + min-width: 100px; + box-shadow: 0px 8px 16px 0px #a8a8a8; + z-index: 1; + padding-top: 0px; + padding-bottom: 0px; + border: solid 1px #404040; + border-radius: 5px; + margin-top: 5px; + margin-left: 14px; + margin-right: 10px; +} + +.dropdown-content-button { + display: none; + position: absolute; + background-color: #ffffff; + min-width: 200px; + max-width: 200px; + box-shadow: 0px 8px 16px 0px #a8a8a8; + z-index: 1; + padding-top: 0px; + padding-bottom: 0px; + border: solid 1px #b8b8b8; + border-top: 0; + border-radius: 0px; + margin-top: 0px; + margin-right: 10px; + font-size: 13px; + overflow-y: auto !important; +} + +.dropdown-content-button a, +.dropdown-content a { + color: black; + text-decoration: none; + display: block; + text-align: left; +} + +.dropdown-content-button a:hover, +.dropdown-content a:hover { + background-color: #f1f1f1; +} + +.dropdown:hover .dropdown-content-button, +.dropdown:hover .dropdown-content { + display: block; +} + +/* is this used? */ +.dropdown:hover a { + color: #00ff00; +} + +.dropdown:hover .dropdown-content-button a, +.dropdown:hover .dropdown-content a { + color: #212529; +} + +.dropdown:hover .dropdown-content-button a:hover, +.dropdown:hover .dropdown-content a:hover { + background-color: #000000; + color: #ffffff !important; +} + +/* Handles nested dropdown entries */ +.dropdown:hover .dropdown-content-button .turnip a, +.dropdown:hover .dropdown-content .turnip a { + color: #212529; +} + +.dropdown:hover .dropdown-content-button .turnip a:hover, +.dropdown:hover .dropdown-content .turnip a:hover { + color: #ffffff !important; +} + +.dropdown:hover .dropdown-content-button .turnip a:hover::before, +.dropdown:hover .dropdown-content .turnip a:hover::before { + color: #ffffff !important; +} + +.dropdown-header { + background-color: #c0e6ee; + color: #000000; + font-weight: 600; + padding: 5px 10px 5px 14px; +} + +hr.dropdown { + border-top: 0; + margin-top: 0px; + margin-bottom: 0px; +} + +hr.divider { + border-top: solid 1px #cccccc; + margin-top: 0px; + margin-bottom: 0px; +} + +hr.divider01 { + border-top: solid 1px #000000; + border-top: solid 1px #d8d8d8; + margin-top: 0px; + margin-bottom: 0px; +} + +.caret { + width: 0; + height: 0; + display: inline-block; + border: 10px solid transparent; + border: 5px solid transparent; + vertical-align: text-bottom; + margin-left: 2px; +} + +.caret.down { + border-top-color: #0066ff; +} + +.dropdown:hover .caret.down { + border-top-color: #ffffff; +} + +.caret.right { + border-left-color: black; +} + +.caret.up { + border-bottom-color: black; +} + +.caret.left { + border-right-color: black; +} + +/* link button */ +.btnLink { + padding-top: 0px; + padding-bottom: 0px; + margin-top: 0px; + display: inline-block; + color: white; + background-color: transparent; + text-align: center; + text-decoration: none; + vertical-align: top; + border: 0; + font-family: "Lato"; + font-size: 13px; +} + +.ansibleNav ul li a.btnLink:hover { + background-color: #000000; +} + +a.btnLink, +.ansibleNav ul li a.btnLink { + color: #ffffff; + text-decoration: none; + padding: 15px 16px 14px 16px; +} + +/* ================================================================= */ + +/* this is the container for the pages */ +.wy-nav-content { + max-width: 100%; + padding-right: 40px; + padding-left: 0; + padding-top: 0; + padding-bottom: 0; + margin-top: 0px; +} + +.wy-nav-content-wrap { + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; + border-right: solid 1px #e8e8e8; +} + +.wy-nav-content { + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; +} + +/* this is the page itself */ +div.rst-content { + background-color: #ffffff; + max-width: 1300px; + background-color: #ffffff; + background-color: #f9f9f9; + background-color: #fcfcfc; + box-shadow: none; + margin-left: 0px; + border: 0; + padding: 0px 80px 10px 80px; + margin-left: 50px; +} + +/* breadcrumbs */ +.wy-breadcrumbs { + height: 10px; +} + +div.rst-breadcrumbs-buttons { + padding-bottom: 10px; + margin-bottom: 10px; + border-bottom: solid 1px #d8d8d8; +} + +/* section [id] .wy-table-responsive, +section [id] .contents { */ +section [id] .wy-table-responsive { + padding-top: 0; + padding-top: 10px; + margin-top: 0; + margin-top: -10px; +} + + + +/* affects bottom padding of page toc */ +.rst-content section ul { + list-style: disc; + line-height: 24px; + margin-bottom: 10px; +} + +/* bullets */ + +.rst-content section ul li::marker { + color: #000000; +} + +.rst-content section ul li a, +.rst-content section ul li a:first-child, +.rst-content section ul li a:last-child { + color: #0066ff; + font-weight: 600; + color: #0033cc; + font-weight: 500; + color: #0066ff; +} + +.rst-content .toctree-wrapper ul li, +.rst-content .toctree-wrapper ul li:first-child, +.rst-content .toctree-wrapper ul li:last-child, +.rst-content section ul li, +.rst-content section ul li:first-child, +.rst-content section ul li:last-child { + list-style: disc; +} + +.rst-content .toctree-wrapper ul li li, +.rst-content .toctree-wrapper ul li li:first-child, +.rst-content .toctree-wrapper ul li li:last-child, +.rst-content section ul li li, +.rst-content section ul li li:first-child, +.rst-content section ul li li:last-child { + list-style: square; +} + +.rst-content .toctree-wrapper ul li li::marker, +.rst-content section ul li li::marker { + color: #000000; + color: #007777; + color: #0066ff; + font-weight: 600; + /* zzz 3/31/2022 Thursday 5:56:36 PM. */ + font-weight: 500; +} + +.rst-content .toctree-wrapper ul li li li, +.rst-content .toctree-wrapper ul li li li:first-child, +.rst-content .toctree-wrapper ul li li li:last-child, +.rst-content section ul li li li, +.rst-content section ul li li li:first-child, +.rst-content section ul li li li:last-child { + list-style: circle; +} + +.rst-content .toctree-wrapper ul li li li::marker, +.rst-content section ul li li li::marker { + color: #000000 !important; + color: #007777 !important; + color: #0066ff !important; + font-weight: 500; +} + + + +.rst-content section .toctree-wrapper ul li.toctree-l1 a { + color: #0066ff; + font-weight: 600; + color: #0033cc; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 a { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 a:hover { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; + text-decoration: underline; + ; +} + +.rst-content section .toctree-wrapper ul li.toctree-l1::marker { + color: #0066ff; + color: #0033cc; + /* zzz 3/31/2022 Thursday 5:50:12 PM. */ + font-weight: 500; + font-weight: 600; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5 li.toctree-l6::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5 li.toctree-l6 li.toctree-l7::marker { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; + /* zzz 3/31/2022 Thursday 6:01:44 PM. */ + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; + font-weight: 500; +} + +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5 li.toctree-l6::marker, +.rst-content section .toctree-wrapper ul li.toctree-l1 li.toctree-l2 li.toctree-l3 li.toctree-l4 li.toctree-l5 li.toctree-l6 li.toctree-l7::marker { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; +} + +.rst-content section ul li li a, +.rst-content section ul li li a:first-child, +.rst-content section ul li li a:last-child, +.rst-content section ul li li li a, +.rst-content section ul li li li a:first-child, +.rst-content section ul li li li a:last-child { + color: #0066ff; + font-weight: 600; + color: #0033cc; + font-weight: 500; + color: #0066ff; +} + +/* the page toc */ +.rst-content section .contents ul li a, +.rst-content section .contents ul li a:first-child, +.rst-content section .contents ul li a:last-child { + color: #0066ff; + font-weight: 600; + color: #0033cc; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .contents ul li li a, +.rst-content section .contents ul li li a:first-child, +.rst-content section .contents ul li li a:last-child { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .contents ul li p, +.rst-content section .contents ul li p:first-child, +.rst-content section .contents ul li p:last-child { + line-height: 15px; + /* zzz 3/31/2022 Thursday 6:12:55 PM. */ + line-height: 17px; +} + +.rst-content section .contents ul li::marker { + color: #0066ff; + color: #0033cc; + font-weight: 500; + font-weight: 600; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .contents ul li li::marker { + color: #000000; + color: #007777; + color: #0066ff; + font-weight: 500; + font-size: 13px; + font-size: 15px; + /* 4/1/2022 Friday 9:09:30 AM. zzz reduce toc font size */ + font-size: 14px; +} + +.rst-content section .contents ul li li li::marker { + color: #000000; + color: #007777; + font-weight: 500; + color: #0066ff; +} + +/* end bullets */ + +.rst-content section ul.simple li>*, +.rst-content section ul.simple li ol, +.rst-content section ul.simple li ul { + margin-top: 5px; +} + +.rst-content section ul li>p, +.rst-content section ul li>p:only-child, +.rst-content section ul li>p:only-child:last-child { + margin-bottom: 5px; + /* zzz 3/31/2022 Thursday 6:10:46 PM. */ + margin-bottom: 0px; +} + +.wy-menu-vertical li.toctree-l1 a button.toctree-expand, +.wy-menu-vertical li.toctree-l2 a button.toctree-expand, +.wy-menu-vertical li.toctree-l3 a button.toctree-expand { + color: #cccccc; +} + +.wy-menu-vertical li.toctree-l1 a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l2 a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l3 a:hover button.toctree-expand { + color: #ffffff; +} + +/* Handle selected entry in the sidebar */ +.wy-menu-vertical li.toctree-l1.current a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l2.current a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l3.current a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l4.current a:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l5.current a:hover button.toctree-expand { + color: #ffffff; +} + +.wy-menu-vertical li.toctree-l1.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l2.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l3.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l4.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l5.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l6.current>a.reference.internal.current button.toctree-expand, +.wy-menu-vertical li.toctree-l7.current>a.reference.internal.current button.toctree-expand { + color: #ffffff; +} + +.wy-menu-vertical li.toctree-l1.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l2.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l3.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l4.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l5.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l6.current>a.reference.internal.current:hover button.toctree-expand, +.wy-menu-vertical li.toctree-l7.current>a.reference.internal.current:hover button.toctree-expand { + color: #ffffff; +} + +/* 3/30/2022 Wednesday 8:41:52 AM. bullet indent? */ +.rst-content .toctree-wrapper ul li, +.rst-content section ul li { + margin-left: 17px; +} + + + +.btn-neutral, +.btn-neutral:hover, +.btn-neutral:visited { + /* color: #404040 !important; */ + /* color: #ff0000 !important; */ + + color: #e60000 !important; + color: #00b359 !important; + color: #00994d !important; + color: #000000 !important; + color: #0066ff !important; + /* color: #ffffff !important; */ + + /* border: 0 !important; */ + border-color: #e8e8e8; + border-color: #e2e2e2; + border-color: #d8d8d8; + border-color: #ffcc00; + border-color: #000000; + /* border-color: #00e673; */ + border-color: #0066ff; + /* border-color: #ff0000; */ + + /* border-color: #c8c8c8; */ + border-radius: 5px; + + box-shadow: none !important; + /* box-shadow: 0 0 3px 3px #e8e8e8 !important; */ + /* box-shadow: 0 0 2px 2px #e8e8e8 !important; */ + + background-color: #ffffff !important; + /* background-color: #f8f8f8 !important; */ + /* background-color: #ffc857 !important; */ + /* background-color: #ffcc00 !important; */ + /* background-color: #00e673 !important; */ + /* background-color: #202020 !important; */ + + text-decoration: none !important; + + padding: 0px 0px 0px 0px; + padding: 5px 5px 4px 5px; + /* padding: 5px; */ +} + +.btn-neutral:hover { + /* color: #000000 !important; */ + /* color: #ffffff !important; */ + /* color: #ff0000 !important; */ + + color: #e60000 !important; + color: #00994d !important; + color: #000000 !important; + /* color: #0066ff !important; */ + /* color: #00b359 !important; */ + /* color: #00994d !important; */ + /* color: #00e673 !important; */ + + border-color: #b8b8b8; + border-color: #cccccc; + border-color: #ff9999; + /* border-color: #ff0000; */ + border-color: #00994d; + border-color: #000000; + /* border-color: #0066ff; */ + /* border-color: #00e673; */ + /* border-color: #00994d; */ + + box-shadow: 0 0 2px 2px #d8d8d8 !important; + box-shadow: 0 0 2px 2px #e8e8e8 !important; + /* box-shadow: 0 0 3px 3px #f2f2f2 !important; */ + + /* background-color: #ff0000 !important; */ + /* background-color: #00e673 !important; */ + background-color: #ffffff !important; +} + +.btn-neutral:focus { + outline: none !important; +} + +/* heading tags */ + +/* h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} */ + + +/* ================================================================= */ + +h1, +.rst-content h1 .toc-backref { + font-size: 27px; + /* font-size: 23px; */ + font-weight: 500; + font-weight: 600; + color: #2952a3; + color: #0066ff; + color: #009933; + color: #ff0066; + color: #cc0066; + color: #ff6666; + color: #ff0000; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + +h2, +.rst-content h2 .toc-backref { + font-size: 26px; + font-size: 22px; + font-weight: 500; + font-weight: 600; + color: #990000; + color: #cc0000; + color: #cc0066; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + + +section [id] h2 { + border-bottom: solid 5px #e8e8e8; + border-bottom: solid 2px #e8e8e8; + border-bottom: solid 2px #009999; + border-bottom: solid 2px #000000; + /* border-bottom: solid 2px #ff3333; */ + /* border-bottom: solid 2px #009999; */ + /* border-bottom: solid 2px #009900; */ + + padding-bottom: 5px; +} + +h3, +.rst-content h3 .toc-backref { + font-size: 25px; + font-size: 21px; + font-weight: 500; + color: #e60000; + color: #e60073; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + +h4, +.rst-content h4 .toc-backref { + font-size: 22px; + font-size: 19px; + font-weight: 500; + color: #ff0000; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + +h5, +.rst-content h5 .toc-backref { + font-size: 21px; + font-size: 18px; + font-weight: 500; + color: #ff3333; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + +h6, +.rst-content h6 .toc-backref { + font-size: 20px; + font-size: 17px; + font-weight: 500; + color: #ff6666; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + +h7, +.rst-content h7 .toc-backref { + font-size: 19px; + font-size: 16px; + font-weight: 500; + color: #ff9999; + /* 4/6/2022 Wednesday 5:36:07 PM. */ + color: #000000; +} + + + + + +/* section [id] h1, */ +section [id] h2, +section [id] h3, +section [id] h4, +section [id] h5, +section [id] h6, +section [id] h7 { + /* padding-top:45px; + margin-top:-45px; + + padding-top:50px; + margin-top:-50px; + + padding-top:60px; + margin-top:-60px; + margin-top:-50px; + + padding-top:20px; + margin-top:-20px; + + padding-top: 50px; + margin-top: -50px; */ + + padding-top: 45px; + padding-top: 55px; + margin-top: -45px; + + /* margin: 0 0 24px; */ +} + +section [id] h2 { + padding-top: 60px; + margin-top: -60px; + margin-top: -30px; +} + + +h1, +.rst-content h1 .toc-backref { + font-family: "Roboto", arial, sans-serif; + font-family: arial; + color: #009900; + color: #b30047; + color: #202020; + color: #ff2222; + color: #009999; + color: #009900; + color: #000000; + /* padding-top: 10px; */ + padding-top: 10px; + font-weight: 600; + /* font-weight: 500; */ +} + +h2, +h3, +h4, +h5, +h6, +h7, +.rst-content h2 .toc-backref, +.rst-content h3 .toc-backref, +.rst-content h4 .toc-backref, +.rst-content h5 .toc-backref, +.rst-content h6 .toc-backref, +.rst-content h7 .toc-backref { + font-family: "Roboto", arial, sans-serif; + font-family: arial; + color: #000000; + color: #009900; + color: #b30047; + color: #202020; + color: #ff3333; + color: #009999; + color: #009900; + color: #000000; + font-weight: 600; + /* font-weight: 500; */ +} + +h4, +.rst-content h4 .toc-backref { + color: #cc0000; + color: #009900; + color: #666666; + color: #006666; + color: #005555; + color: #3366cc; + color: #86592d; + color: #734d26; + color: #006080; + color: #336699; + color: #2d5986; + color: #009900; + color: #cd0000; + color: #990000; + /* color: #ff0000; */ + /* color: #ff4444; */ + /* color: #ff3333; */ + /* color: #007777; */ + /* color: #666699; */ + /* color: #993399; */ + color: #3333cc; + color: #000000; + color: #555555; + color: #3366cc; + color: #0066cc; + /* color: #006666; */ + /* color: #336699; */ + /* color: #666666; */ + + font-weight: 500; + font-weight: 600; + /* font-style: italic; */ + /* 4/6/2022 Wednesday 5:36:07 PM. */ + /* font-style: normal; */ +} + +h5, +h6, +h7, +.rst-content h5 .toc-backref, +.rst-content h6 .toc-backref, +.rst-content h7 .toc-backref { + font-family: "Roboto", arial, sans-serif; + font-family: arial; + color: #000000; + color: #555555; + color: #3366cc; + color: #0066cc; + /* color: #006666; */ + /* color: #336699; */ + /* color: #666666; */ + font-weight: 600; + /* font-weight: 500; */ + /* font-style: italic; */ + /* 4/6/2022 Wednesday 5:36:07 PM. */ + /* font-style: normal; */ +} + +/* .rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6, .rst-content h7 { + margin-bottom: 24px; + margin-bottom: 15px; +} */ + +/* .rst-content h2 { + margin-bottom: 10px; + margin-bottom: 20px; +} */ + + + +/* 9/17/2021 Friday 7:14:49 PM. This removes the hover underline from the headers. and every other link on the page!*/ +.wy-nav-content-wrap a:hover { + text-decoration: underline; + /* text-decoration: none; */ +} + + + +.wy-nav-content-wrap h1 a:hover, +.wy-nav-content-wrap h2 a:hover, +.wy-nav-content-wrap h3 a:hover, +.wy-nav-content-wrap h4 a:hover, +.wy-nav-content-wrap h5 a:hover, +.wy-nav-content-wrap h6 a:hover, +.wy-nav-content-wrap h7 a:hover, +/* don't these also need the prefix added above? */ +.rst-content h1 .toc-backref a:hover, +.rst-content h2 .toc-backref a:hover, +.rst-content h3 .toc-backref a:hover, +.rst-content h4 .toc-backref a:hover, +.rst-content h5 .toc-backref a:hover, +.rst-content h6 .toc-backref a:hover, +.rst-content h7 .toc-backref a:hover { + text-decoration: none; +} + + +h6, +.rst-content h6 .toc-backref { + font-weight: 500; + font-weight: 600; +} + +h7, +.rst-content h7 .toc-backref { + font-weight: 500; + /* font-weight: 600; */ + /* font-style: italic; */ + + /* padding-top: 60px; */ + /* margin-top: -60px; */ + /* margin-top: -100px; */ + + /* padding-bottom: 0px; */ + /* margin-bottom: 20px; */ + + margin-bottom: 20px; +} + +.rst-content h7 .headerlink { + visibility: hidden; + font-size: 14px; + display: inline-block; + margin-bottom: 10px; +} + +.rst-content h7 .headerlink:hover { + visibility: visible; +} + + +/* .rst-content .toctree-wrapper > p.caption, */ +/* .rst-content h1, */ +/* .rst-content h2, */ +.rst-content h3, +.rst-content h4, +.rst-content h5, +.rst-content h6 { + margin-bottom: 24px; + margin-bottom: 10px; +} + + +/* end heading tags */ + +/* 4/1/2022 Friday 9:39:02 AM. Add bottom padding to "Edit on GitHub" link */ +/* .wy-breadcrumbs-aside > .fa::before { + padding-bottom: 4px; +} */ + + +/* ========================== footer =========================== */ + + +footer { + padding-top: 10px; + padding-top: 0px; + padding-bottom: 10px; + padding-bottom: 20px; + + margin-top: 10px; + margin-top: 30px; + margin-top: 50px; + margin-top: 70px; + + border-top: solid 1px #ff0000; + border-top: solid 1px #e8e8e8; + border-top: solid 1px #d8d8d8; + + font-size: 13px; +} + +footer p { + font-size: 13px; +} + +footer>hr { + border-top: solid 1px #009900; + border-top: solid 1px #e8e8e8; + border-top: solid 1px transparent; + margin: 0px 0px 0px 0px; + margin: 5px 0px 5px 0px; +} + +.rst-footer-buttons { + padding-top: 50px; + padding: 0px 0px 0px 0px; + padding: 0px 0px 10px 0px; + padding: 10px 0px 10px 0px; + + font-size: 14px; +} + + + + + +/* 4/2/2022 Saturday 10:42:26 AM. ================================ consolidate the toc code here */ + +/* page toc -- 4/2/2022 Saturday 11:44:41 AM. No longer used for page toc */ +.rst-content .topic-title { + font-size: 14px; + font-weight: 500; + color: #202020; + margin-top: 10px; + margin-bottom: 0px; + border-bottom: 0; + padding: 0px 10px 5px 0px; +} + +/* 4/2/2022 Saturday 1:21:45 PM. No longer being used */ +.toc { + color: #808080; + font-size: 18px; + font-family: Arial; + font-weight: 500; + background-color: #00e673; +} + +/* title section of page/section/entire RTD outline */ +.div_page_outline, +.div_section_outline, +.div_rtd_outline { + font-size: 16px; + font-family: Lato, proxima-nova, Helvetica Neue, Arial, sans-serif; + border-radius: 5px 5px 0px 0px; + border: solid 1px #000000; + border-bottom: 0px; + color: #000000; + /* color: #990000; */ + + background: #99ffcc; + /* background: #b3ffd9; */ + /* background: #ccffe6; */ + background: #0066ff; + background: #f8f8f8; + /* background: #ffffff; */ + background: #f5f5f5; + background-color: #f8f8f8; + + border-color: #d8d8d8; + border-color: #e8e8e8; + border-color: #e2e2e2; + /* border-color: #0066ff; */ + /* border-color: #202020; */ + /* border: 0; */ + + padding: 10px 10px 7px 15px; + padding: 7px 10px 4px 15px; + padding: 7px 10px 4px 5px; + + font-weight: 600; + font-weight: 500; + line-height: 15px; + display: block; +} + + +/* page outline */ +section [id] .contents, +.contents { + border: solid 1px #cccccc; + border-color: #d8d8d8; + border-color: #e8e8e8; + border-color: #e2e2e2; + /* border: 0; */ + + border-top: solid 1px #000000; + border-top: solid 2px #0066ff; + border-top: solid 2px #000000; + /* border-top: solid 1px #990000; */ + border-radius: 0px 0px 5px 5px; + background-color: #f8f8f8; + /* background-color: #ffffff; */ + background-color: #f5f5f5; + background-color: #f8f8f8; + padding: 10px 10px 0px 10px; + padding-bottom: -30px; + margin-top: -10px; + margin-bottom: 30px; + box-shadow: #d8d8d8 0px 2px 4px 0px, #d8d8d8 0px 2px 16px 0px; + box-shadow: none; +} + + +/* entire RTD/section outline */ +.toctree-wrapper { + background-color: #f8f8f8; + /* background-color: #ffffff; */ + background-color: #f5f5f5; + background-color: #f8f8f8; + border: solid 1px #cccccc; + border-color: #d8d8d8; + border-color: #e8e8e8; + border-color: #e2e2e2; + /* border: 0; */ + + border-top: solid 1px #000000; + border-top: solid 2px #0066ff; + border-top: solid 2px #000000; + /* border-top: solid 1px #990000; */ + border-radius: 0px 0px 5px 5px; + padding: 10px; + padding-bottom: 0px; + margin-top: -10px; + margin-bottom: 30px; + box-shadow: #d8d8d8 0px 2px 4px 0px, #d8d8d8 0px 2px 16px 0px; + box-shadow: none; +} + +.blue-background { + background-color: #ccffff; + background-color: #b3ffff; + color: #000000; + padding: 3px 0px 3px 0px; +} + +.green-background { + background-color: #ccffcc; + color: #000000; + padding: 3px 0px 3px 0px; +} + +.yellow-background { + background-color: #ffff33; + background-color: #ffff99; + background-color: #ffffb3; + color: #000000; + padding: 3px 0px 3px 0px; +} + +.red-background { + background-color: #ffcccc; + color: #000000; + padding: 3px 0px 3px 0px; +} \ No newline at end of file diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000..5863ccf5 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,26 @@ +/* this is the container for the pages */ +.wy-nav-content { + max-width: 100%; + padding: 0px 40px 0px 0px; + margin-top: 0px; +} + +.wy-nav-content-wrap { + border-right: solid 1px; +} + +div.rst-content { + max-width: 1300px; + border: 0; + padding: 10px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } +} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..b6aafba9 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,117 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = "aboutcode-toolkit" +copyright = "nexB Inc. and others." +author = "AboutCode.org authors and contributors" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.intersphinx", + "sphinx_reredirects", + "sphinx_rtd_theme", + "sphinx_rtd_dark_mode", + "sphinx.ext.extlinks", + "sphinx_copybutton", +] + + +# Redirects for olds pages +# See https://documatt.gitlab.io/sphinx-reredirects/usage.html +redirects = {} + +# This points to aboutcode.readthedocs.io +# In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot +# Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 + +intersphinx_mapping = { + "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), + "scancode-workbench": ( + "https://scancode-workbench.readthedocs.io/en/develop/", + None, + ), +} + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +master_doc = "index" + +html_context = { + "display_github": True, + "github_user": "nexB", + "github_repo": "aboutcode-toolkit", + "github_version": "develop", # branch + "conf_py_path": "/docs/source/", # path in the checkout to the docs root +} + +html_css_files = [ + "theme_overrides.css", +] + + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +html_show_sphinx = True + +# Define CSS and HTML abbreviations used in .rst files. These are examples. +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` +rst_prolog = """ +.. |psf| replace:: Python Software Foundation + +.. # define a hard line break for HTML +.. |br| raw:: html + +
    + +.. role:: red + +.. role:: img-title + +.. role:: img-title-para + +""" + +# -- Options for LaTeX output ------------------------------------------------- + +latex_elements = {"classoptions": ",openany,oneside"} diff --git a/docs/source/general.rst b/docs/source/general.rst new file mode 100644 index 00000000..35232b7b --- /dev/null +++ b/docs/source/general.rst @@ -0,0 +1,515 @@ +.. _general: + +======= +General +======= + +AboutCode Toolkit Defined +========================= + +AboutCode Toolkit is a tool for your software development team to document your code +inside your codebase, typically in preparation for a product release, side-by-side with the +actual code. ABOUT file(s) have a simple, standard format that identifies components and their +associated licenses. The current AboutCode Toolkit subcommands are: + +- **attrib**: Generate a Product Attribution notice document from your ABOUT + file(s), JSON, CSV or XLSX. You can also generate documents for other + purposes (such as a License Reference) by varying your input control file + and your template. + +- **check**: A simple command to validate the ABOUT file(s) and output + errors/warnings on the terminal. + +- **collect_redist_src**: A command to collect and copy sources that have + the 'redistribute' flagged as 'True' in ABOUT file(s) or from an inventory. + +- **gen**: Create ABOUT file(s) from a Software Inventory file (.csv, .json or .xlsx format) + which is typically created from a software audit, and insert these AboutCode Toolkit files + into your codebase. You can regenerate the AboutCode Toolkit files from a new + Software Inventory file whenever you make changes. + +- **gen_license**: Fetch licenses in the license_expression field and + save to the output location. + +- **inventory**: Generate a Software Inventory list (.csv, .json or .xlsx format) + from your codebase based on ABOUT file(s). Note that this Software Inventory will + only include components that have AboutCode Toolkit data. In another word, if you do + not create AboutCode Toolkit files for your own original software components, + these components will not show up in the generated inventory. + +- **transform**: A command to transform an input CSV/JSON/XLSX by applying + renaming and/or filtering and then output to a new CSV/JSON/XLSX file. + +Additional AboutCode Toolkit information is available at: + +- See :ref:`specification` for an overview and a link to the ABOUT File specification. + +- https://github.com/aboutcode-org/aboutcode-toolkit/ for the AboutCode Toolkit tools. + +Key Terminology +=============== +Some key terminology that applies to AboutCode Toolkit tool usage: + +- **Software Inventory or Inventory** - means a list of all of the components + in a Development codebase and the associated data about those components with a + focus on software pedigree/provenance- related data for open source and + third-party components. + +- **Product BOM or BOM** - means a subset list of the components in a Development + codebase (Software Inventory) that are Deployed on a particular Product + Release (a Product Bill of Materials). + +Using gen to Generate ABOUT file(s) +=================================== + +Prepare Your Software Inventory for gen Standard Field Names +------------------------------------------------------------ + +You should start with a software inventory of your codebase in spreadsheet or JSON format. You need +to prepare a version of it that will identify the field values that you want to appear +in your .ABOUT files. Note the following standard field names (defined in the ABOUT +File Specification), which gen will use to look for the values that it will store in your +generated .ABOUT files, as well as any additional text files that you identify, which +it will copy and store next to the .ABOUT files. + +.. list-table:: + :widths: 10 45 45 + :header-rows: 1 + + * - Standard Field Name + - Description + - Notes + * - name + - Component name + - Mandatory + * - about_resource + - Name/path of the component resource + - Optional + * - ignored_resources + - List of paths ignored from the ``about_resource`` + - Optional + * - version + - Component version + - Optional + * - download_url + - Direct URL to download the original file or archive documented by this ABOUT file + - Optional + * - description + - Component description + - Optional + * - homepage_url + - URL to the homepage for this component + - Optional + * - package_url + - Package URL for this component (See https://github.com/package-url/purl-spec for SPEC) + - Optional + * - notes + - notes text + - Optional + * - license_expression + - Expression for the license of the component using ScanCode license key(s). + - Optional. You can separate each identifier using " OR " and " AND " to document the relationship between multiple license identifiers, such as a choice among multiple licenses. + * - license_key + - ScanCode license key for the component. + - Optional. gen will obtain license information from ScanCode LicenseDB or DejaCode Enterprise if the --fetch-license or --fetch-license-djc option is set, including the license text, in order to create and write the appropriate .LICENSE file in the .ABOUT file target directory. + * - license_name + - License name for the component. + - Optional. This field will be generated if the --fetch-license or --fetch-license-djc option is set. + * - license file + - license file name + - Optional. gen will look for the file name (if a directory is specified in the --reference option) to copy that file to the .ABOUT file target directory. + * - license_url + - URL to the license text for the component + - Optional + * - spdx_license_key + - The ScanCode LicenseDB spdx_license_key defined for the license at https://scancode-licensedb.aboutcode.org/index.html + - Optional + * - declared_license_expression + - A license expression derived from statements in the manifests or key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files. + - Optional + * - other_license_expression + - A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files. + - Optional + * - copyright + - copyright statement for the component + - Optional + * - notice_file + - notice text file name + - Optional + * - notice_url + - URL to the notice text for the component + - Optional + * - redistribute + - Yes/No. Does the component license require source redistribution. + - Optional + * - attribute + - Yes/No. Does the component license require publishing an attribution or credit notice. + - Optional + * - track_changes + - Yes/No. Does the component license require tracking changes made to the component. + - Optional + * - modified + - Yes/No. Have the component been modified. + - Optional + * - internal_use_only + - Yes/No. Is the component internal use only. + - Optional + * - changelog_file + - changelog text file name + - Optional + * - owner + - name of the organization or person that owns or provides the component + - Optional + * - owner_url + - URL to the owner for the component + - Optional + * - contact + - Contact information + - Optional + * - author + - author of the component + - Optional + * - author_file + - author text file name + - Optional + * - vcs_tool + - Name of the version control tool. + - Optional + * - vcs_repository + - Name of the version control repository. + - Optional + * - vcs_path + - Name of the version control path. + - Optional + * - vcs_tag + - Name of the version control tag. + - Optional + * - vcs_branch + - Name of the version control branch. + - Optional + * - vcs_revision + - Name of the version control revision. + - Optional + * - checksum_md5 + - MD5 value for the file + - Optional + * - checksum_sha1 + - SHA1 value for the file + - Optional + * - checksum_sha256 + - SHA256 value for the file + - Optional + * - spec_version + - The version of the ABOUT file format specification used for this file. + - Optional + + +Fields Renaming and Optional Custom Fields +------------------------------------------ + +Since your input's field name may not match with the AboutCode Toolkit standard field name, +you can use the transform subcommand to do the transformation. + +A transform configuration file is used to describe which transformations and validations to +apply to a source CSV/JSON/XLSX file. This is a simple text file using YAML format, +using the same format as an .ABOUT file. + +The attributes that can be set in a configuration file are: + +- field_renamings: An optional map of source field name to target new field + name that is used to rename CSV/JSON/XLSX fields. + + .. code-block:: + + field_renamings: + about_resource : 'Directory/Location' + bar : foo + + +The renaming is always applied first before other transforms and checks. All other +field names referenced below are AFTER the renaming have been applied. +For instance with this configuration, the field "Directory/Location" will be +renamed to "about_resource" and "foo" to "bar": + +- required_fields: An optional list of required field names that must have a value, + beyond the standard field names. If a source CSV/JSON/XLSX does not have such a field or + an entry is missing a value for a required field, an error is reported. + +For instance with this configuration, an error will be reported if the fields "name" +and "version" are missing, or if any entry does not have a value set for these fields: + + .. code-block:: + + required_fields: + - name + - version + +- field_filters: An optional list of fields that should be kept in the transformed file. + If this list is provided, only the fields that are in the list will be kept. All others will + be filtered out even if they are AboutCode Toolkit standard fields. If this list is not + provided, all source fields are kept in the transformed target file. + +For instance with this configuration, the target file will only contains the "name" and +"version" fields: + + .. code-block:: + + field_filters: + - name + - version + +- exclude_fields: An optional list of field names that should be excluded in the transformed + file. If this list is provided, all the fields from the source file that should be + excluded in the target file must be listed. Excluding required fields will cause an error. + If this list is not provided, all source fields are kept in the transformed target file. + +For instance with this configuration, the target file will not contain the "type" and "temp" fields: + + .. code-block:: + + exclude_fields: + - type + - temp + +Run gen to Generate ABOUT file(s) +--------------------------------- + +When your software inventory is ready, you can save it as a .csv, .json or .xlsx file, +and use it as input to run gen to generate ABOUT file(s). The official gen parameters +are defined here: :ref:`reference` + +Here is an example of a gen command: + + .. code-block:: + + about gen --fetch-license --reference /Users/harrypotter/myLicenseNoticeFiles/ /Users/harrypotter/myAboutFiles/myProject-bom.csv /Users/harrypotter/myAboutFiles/ + +This gen example command does the following: + +- Activates the --fetch-license option to get license information from ScanCode LicenseDB. + +- Activates the --reference option to get license text files and notice text files that + you have specified in your software inventory to be copied next to the + associated .ABOUT files when those are created. + +- Specifies the path of the software inventory to control the processing. + +- Specifies a target output directory. + +Review the generated ABOUT file(s) to determine if it meets your requirements. Here is a +simple example of a linux-redhat-7.2.ABOUT file that documents the directory /linux-redhat-7.2/ : + + .. code-block:: + + about_resource: . + name: Linux RedHat + version: v 7.2 + attribute: Y + copyright: Copyright (c) RedHat, Inc. + license_expression: gpl-2.0 + licenses: + - key: gpl-2.0 + name: GPL 2.0 + file: gpl-2.0.LICENSE + url: https://scancode-licensedb.aboutcode.org/gpl-2.0.LICENSE + spdx_license_key: GPL-2.0-only + owner: Red Hat + redistribute: Y + +You can make appropriate changes to your input software inventory and then run +gen as often as necessary to replace the ABOUT file(s) with the improved version. + +Using attrib to Generate a Product Attribution Notice Package +============================================================= + +Prepare an Attribution Template to Use +-------------------------------------- + +You can run attrib using the default_html.template (or default_json.template) +provided with the AboutCode Toolkit tools: + +https://github.com/aboutcode-org/aboutcode-toolkit/blob/develop/src/attributecode/templates/default_html.template + +If you choose to do that, you will most likely want to edit the generated .html +file to provide header information about your own organization and product. + +Running attrib with the default_html.template file is probably your best choice when +you are still testing your AboutCode Toolkit process. Once you have a good understanding +of the generated output, you can customize the template to provide the standard text that +serve your needs. You can also create alternative versions of the template to use attrib +to generate other kinds of documents, such as a License Reference. + +Use jinja2 Features to Customize Your Attribution Template +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The attrib tool makes use of the open source python library jinja2 +(http://jinja.pocoo.org/docs/dev/templates/) in order to extend .html capabilities and +transform AboutCode Toolkit input data into the final format of the generated attribution +file. ``default_html.template`` file contains text that complies with jinja2 syntax specifications +in order to support grouping, ordering, formatting and presentation of your AboutCode +Toolkit data. If your attribution requirements are complex, you may wish to study the jinja2 +documentation to modify the default_html.template logic or create your own template; alternatively, +here are a few relatively simple concepts that relate to the attribution document domain. + +The simplest modifications to the default_html.template file involve the labels and standard +text. For example, here is the default template text for the Table of Contents: + + .. code-block:: + +
    + {% for about_object in abouts %} +

    {{ about_object.name.value }} + {% if about_object.version.value %} {{ about_object.version.value }} + {% endif %}

    + {% endfor %} +
    + +If you would prefer something other than a simple space between the component name and +the component version, you can modify it to something like this: + + .. code-block:: + +
    + {% for about_object in abouts %} +

    {{ about_object.name.value }} + {% if about_object.version.value %} - Version {{ about_object.version.value }} + {% endif %}

    + {% endfor %} +
    + +The ``if about_object.version.value`` is checking for a component version, and if one +exists it generates output text that is either a space followed by the actual version +value, or, as in this customized template, it generates output text as " - Version ", +followed by the actual version value. You will, of course, want to test your output to +get exactly the results that you need. + +Note that you can actually use attrib to generate an AboutCode Toolkit-sourced document +of any kind for varying business purposes, and you may want to change the grouping/ordering +of the data for different reporting purposes. (Here we get into somewhat more complex usage of +jinja2 features, and you may wish to consult the jinja2 documentation to reach a more comprehensive +understanding of the syntax and features.) The default ordering is by component, but In the +following example, which is intended to support a "license reference" rather than an attribution +document, the customized template modifies the data grouping to use a custom field +called "confirmed_license": + + .. code-block:: + +
    + {% for group in abouts | groupby('confirmed_license') %} +

    + {% for license in group.grouper.value %} + {{ license }} + + {% endfor %} +

    + {% endfor %} +
    + +After the table of contents, this example customized template continues with the license details +using the jinja2 for-loop capabilities. Notice that the variable "group.grouper.value" is +actually the license name here, and that “License URL” can be any URL that you have chosen +to store in your .ABOUT files: + + .. code-block:: + + {% for group in abouts | groupby('confirmed_license') %} + {% for confirmed_license in group.grouper.value %} + +
    +

    {{ confirmed_license }}

    +

    This product contains the following open source software packages licensed under the terms of the license: {{confirmed_license}}

    + +
    + {%for about_object in group.list %} + {% if loop.first %} + {% if about_object.license_url.value %} + {% for lic_url in about_object.license_url.value %} +

    License URL: {{lic_url }}

    + {% endfor %} + {% endif %} + {% endif %} +
  • + {{ about_object.name.value }}{% if about_object.version.value %} - Version + {{ about_object.version.value }}{% endif %} +
    • + {% if about_object.copyright.value %}
    {{about_object.copyright.value}}
    {% endif %} + {% if loop.last %} + 
    +                            {% for lic_key in about_object.license_file.value %}
+                                {{about_object.license_file.value[lic_key]}}
+                            {% endfor %}
+
    + {% endif %} + {% endfor %} +
    +
    +
    + {% endfor %} + {% endfor %} +
    + +In summary, you can start with simple, cosmetic customizations to the default_html.template, +and gradually introduce a more complex structure to the attrib output to meet +varying business requirements. + +Run attrib to Generate a Product Attribution Notice Package +----------------------------------------------------------- + +You can then run the attrib to generate your product attribution notice package from the +generated ABOUT file(s) or from an inventory (.csv/.json/.xlsx). The official attrib +parameters are defined here: :ref:`reference` + +Here is an example of a attrib command: + +``about attrib --template /Users/harrypotter/myAboutFiles/my_attribution_template_v1.html +/Users/harrypotter/myAboutFiles/ /Users/harrypotter/myAboutFiles +/myProject-attribution-document.html`` + +Note that this example attrib command does the following: + +- Activates the ``--template`` option to specify a custom output template. + +- Specifies the path of the ABOUT file(s) that use to generate the output attribution. + +- Specifies the full path (include file name) of the output document to be generated. + +Another example: + +``about attrib /Users/harrypotter/inventory.xlsx +/Users/harrypotter/attribution.html --reference /Users/harrypotter/licenses/`` + +The above command does the following: + +- Use the ``inventory.xlsx`` as the input + +- Specifies the location of the generated output document + +- Specifies the licesen_file or notice_file location that can be found in the + ``--reference`` option + + +A successful execution of attrib will create a .html (or .json depends on the template) +file that is ready to use to meet your attribution requirements. + +Please refer to the ``attrib`` section in :ref:`reference` for more information. + +Using inventory to Generate a Software Inventory +================================================ + +Generate a Software Inventory of Your Codebase from ABOUT file(s) +----------------------------------------------------------------- + +One of the major features of the ABOUT File specification is that the .ABOUT files +are very simple text files that can be created, viewed and edited using any standard +text editor. Your software development and maintenance processes may require or encourage +your software developers to maintain .ABOUT files and/or associated text files manually. +For example, when a developer addresses a software licensing issue with a component, +it is appropriate to adjust the associated ABOUT file(s) manually. + +If your organization adopts the practice of manually creating and maintaining ABOUT file(s), +you can easily re-create your software inventory from your codebase using inventory. +The official inventory parameters are defined here: :ref:`reference` + +A successful execution of inventory will create a complete software inventory in .csv, +.json or .xlsx format based on defined format. diff --git a/docs/source/home.rst b/docs/source/home.rst new file mode 100644 index 00000000..414cc293 --- /dev/null +++ b/docs/source/home.rst @@ -0,0 +1,140 @@ +AboutCode Toolkit +================= + +Introduction +------------ +The AboutCode Toolkit and ABOUT files provide a simple way to document the +origin, license, usage and other important or interesting information about +third-party software components that you use in your project. + +You start by storing ABOUT files (a small YAML formatted text file with field/value pairs) +side-by-side with each of the third-party software components you use. +Each ABOUT file documents origin and license for one software. +There are many examples of ABOUT files (valid or invalid) in the testdata/ +directory of the whole repository. + +The current version of the AboutCode Toolkit can read these ABOUT files so that you +can collect and validate the inventory of third-party components that you use. + +In addition, this tool is able to generate attribution notices and +identify redistributable source code used in your project to help you comply +with open source licenses conditions. + +This version of the AboutCode Toolkit follows the ABOUT specification version 3.3.2 at: +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/specification.html + + +REQUIREMENTS +------------ +The AboutCode Toolkit is tested with Python 3.9 or above only on Linux, Mac and Windows. +You will need to install a Python interpreter if you do not have one already +installed. + +On Linux and Mac, Python is typically pre-installed. To verify which +version may be pre-installed, open a terminal and type: + + python --version + +.. note:: + Debian has decided that distutils is not a core python package, so it is not included + in the last versions of debian and debian-based OSes. + + A solution is to run: `sudo apt install python3-distutils` + +On Windows or Mac, you can download the latest Python here: + https://www.python.org/downloads/ + +Download the .msi installer for Windows or the .dmg archive for Mac. +Open and run the installer using all the default options. + +INSTALLATION +------------ +Checkout or download and extract the AboutCode Toolkit from: + https://github.com/aboutcode-org/aboutcode-toolkit/ + +To install all the needed dependencies in a virtualenv, run (on posix): + ./configure +or on windows: + configure + +ACTIVATE the VIRTUALENV +----------------------- +To activate the virtualenv, run (on posix): + source venv/bin/activate +or on windows: + venv\\bin\\activate + + +DEACTIVATE the VIRTUALENV +------------------------- +To deactivate the virtualenv, run (on both posix and windows): + deactivate + + +VERSIONING SCHEMA +----------------- +Starting at AboutCode version 4.0.0, the AboutCode Toolkit will follow SemVer +for the versioning schema. + +i.e. MAJOR.MINOR.PATCH format + 1. MAJOR version when making incompatible API changes, + 2. MINOR version when making functionality in a backwards compatible manner, and + 3. PATCH version when making backwards compatible bug fixes. + + +REFERENCE +--------- +See https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/ +for documentation. + +See +https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/en/latest/reference.html +for reference. + +TESTS and DEVELOPMENT +--------------------- +To install all the needed development dependencies, run (on posix): + ./configure --dev +or on windows: + configure --dev + +To verify that everything works fine you can run the test suite with: + pytest + + +CLEAN BUILD AND INSTALLED FILES +------------------------------- +To clean the built and installed files, run (on posix): + ./configure --clean +or on windows: + configure --clean + + +HELP and SUPPORT +---------------- +If you have a question or find a bug, enter a ticket at: + + https://github.com/aboutcode-org/aboutcode-toolkit + +For issues, you can use: + + https://github.com/aboutcode-org/aboutcode-toolkit/issues + + +SOURCE CODE +----------- +The AboutCode Toolkit is available through GitHub. For the latest version visit: + + https://github.com/aboutcode-org/aboutcode-toolkit + + +HACKING +------- +We accept pull requests provided under the same license as this tool. +You agree to the https://developercertificate.org/ + + +LICENSE +------- +The AboutCode Toolkit is released under the Apache 2.0 license. +See (of course) the about.ABOUT file for details. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..3c5b53b0 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,14 @@ +================================= +AboutCode Toolkit's Documentation +================================= + +Welcome to the AboutCode Toolkit's Documentation. + +.. toctree:: + :maxdepth: 2 + + AboutCode Toolkit + General + Specification + Reference + Type of Errors diff --git a/docs/source/reference.rst b/docs/source/reference.rst new file mode 100644 index 00000000..e7f8d25d --- /dev/null +++ b/docs/source/reference.rst @@ -0,0 +1,907 @@ +.. _reference: + +========= +Reference +========= + +about +===== + +Syntax +------ + + .. code-block:: + + about [OPTIONS] [COMMANDS] + +Options +------- + + .. code-block:: + + --version Show the version and exit. + -h, --help Show this message and exit. + +Commands +-------- + + .. code-block:: + + attrib Generate an attribution document from + JSON/CSV/XLSX/.ABOUT files. + check Validate that the format of .ABOUT files is correct and + report errors and warnings. + collect-redist-src Collect redistributable sources. + gen Generate .ABOUT files from an inventory as + CSV/JSON/XLSX. + gen-license Fetch and save all the licenses in the + license_expression field to a directory. + inventory Collect the inventory of .ABOUT files to a CSV/JSON/XLSX + file. + transform Transform a CSV/JSON/XLSX by applying renamings, filters + and checks. + +attrib +====== + +Syntax +------ + + .. code-block:: + + about attrib [OPTIONS] LOCATION OUTPUT + + INPUT: Path to a file (.ABOUT/.csv/.json/.xlsx), directory or .zip archive containing .ABOUT files. + + OUTPUT: Path where to write the attribution document. + +Options +------- + + .. code-block:: + + --api_url URL URL to DejaCode License Library. + --api_key KEY API Key for the DejaCode License Library + --min-license-score INTEGER Attribute components that have license score + higher than or equal to the defined --min- + license-score. + --scancode Indicate the input JSON file is from + scancode_toolkit. + --reference DIR Path to a directory with reference files where + "license_file" and/or "notice_file" located. + --template FILE Path to an optional custom attribution template + to generate the attribution document. If not + provided the default built-in template is used. + --vartext = Add variable text as key=value for use in a + custom attribution template. + --worksheet name The worksheet name from the INPUT. (Default: + the "active" worksheet) + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Generate an attribution file which contains license information from the INPUT +along with the license text. + +Assume the following: + + .. code-block:: + + '/home/about_files/' contains all the ABOUT files [INPUT] + '/home/project/inventory.csv' is a BOM inventory [INPUT] + '/home/project/scancode-detection.json' is a detection output from scancode-toolkit[INPUT] + '/home/project/licenses/' contains all the license/notice file references + '/home/attribution/attribution.html' is the user's output path [OUTPUT] + + + .. code-block:: + + $ about attrib /home/about_files/ /home/attribution/attribution.html + or + $ about attrib /home/project/inventory.csv /home/attribution/attribution.html --reference /home/project/licenses/ + or + $ about attrib --scancode /home/project/scancode-detection.json /home/attribution/attribution.html + +Details +^^^^^^^ + + .. code-block:: + + --api_url URL --api_key + + This option let user to define where to get the license information such as + from DJE. If these options are not set, the tool will get the license + information from ScanCode LicenseDB by default + + $ about attrib --api_url --api_key INPUT OUTPUT + + --min-license-score + + This option is a filter to collect license information where the license score + in the scancode toolkit detection is greater than or equal to the defined + --min-license-score. This option is specifically design for scancode's input + and therefore --scancode is required + + $ about attrib --scancode --min-license-score 85 /home/project/scancode-detection.json OUTPUT + + --reference + + This option is to define the reference directory where the 'license_file' + or 'notice_file' are stored + + $ about attrib --reference /home/project/licenses/ /home/project/inventory.csv OUTPUT + + --template + + This option allows you to use your own template for attribution generation. + For instance, if you have a custom template located at: + /home/custom_template/template.html + + $ about attrib --template /home/custom_template/template.html INPUT OUTPUT + + --vartext + + This option allow you to pass variable texts to the attribution template + + $ about attrib --vartext "title=Attribution Notice" --vartext "header=Product 101" LOCATION OUTPUT + + Users can use the following in the template to get the vartext: + {{ vartext['title'] }} + {{ vartext['header'] }} + + --worksheet + + This option identify the worksheet name from the XLSX input to work with. + If no worksheet is defined, the "active" worksheet will be used + + $ about attrib --worksheet BOM /home/project/audit.xlsx OUTPUT + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +The following data are passed to jinja2 and, therefore, can be used for a custom template: + * about object: the about objects + * common_licenses: a common license keys list in licenses.py + * licenses_list: a license object list contains all the licenses found in about objects. + It contains the following attribute: key, name, filename, url, text + +check +===== + +Syntax +------ + + .. code-block:: + + about check [OPTIONS] LOCATION + + LOCATION: Path to an ABOUT file or a directory with ABOUT files. + +Options +------- + + .. code-block:: + + --exclude PATTERN Exclude the processing of the specified input pattern + (e.g. *tests* or test/). + --license Validate the license_expression value in the input. + --djc api_url api_key Validate license_expression from a DejaCode License + Library API URL using the API KEY. + --log FILE Path to a file to save the error messages if any. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Validating ABOUT files at LOCATION. + +Details +^^^^^^^ + + .. code-block:: + + --exclude + Exclude the processing of the specified input pattern + + It takes a pattern or an exact directory as an argument. + Multiple `--exclude` can be used. + + $ about check --exclude ./tests/ --exclude "*sample*" /home/project/about_files/ + + --license + Validate the license_expression value in the input. + + If this option is not flagged, only the basic syntax is checked. + No validation of the license_expression value. + + $ about check --license /home/project/about_files/ + + ---djc + + Validate license_expression from a DejaCode License. + + This option requires 2 parameters: + api_url - URL to the DJE License Library. + api_key - Hash key to authenticate yourself in the API. + + In addition, the input needs to have the 'license_expression' field. + (Please contact nexB to get the api_* value for this feature) + + $ about check --license --djc 'api_url' 'api_key' /home/project/about_files/ + + --log + + This option save the error log to the defined location + + $ about check --log /home/project/error.log /home/project/about_files/ + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + + $ about check --verbose /home/project/about_files/ + +Special Notes +------------- +`--djc` +^^^^^^^ + +If no `--djc` option is set, the tool will default to check license_expression from +ScanCode LicenseDB. + +`--exclude` +^^^^^^^^^^^ + +As the `--exclude` option accepts patterns that may include wildcards, the running shell could +expand these patterns into filenames, potentially causing errors. To avoid this, +ensure the pattern is wrapped in quotes. + +On Windows, users can either use `^` to escape the `*` or use `--%` +before `--exclude` to prevent shell globbing. + +$ about check /home/project/about_files/ --exclude "tests^*" + +$ about check /home/project/about_files/ --% --exclude "*tests*" + +collect_redist_src +================== + +Syntax +------ + + .. code-block:: + + about collect_redist_src [OPTIONS] LOCATION OUTPUT + + LOCATION: Path to a directory containing sources that need to be copied + (and containing ABOUT files if `inventory` is not provided) + + OUTPUT: Path to a directory or a zip file where sources will be copied to. + +Options +------- + + .. code-block:: + + --from-inventory FILE Path to an inventory CSV/JSON/XLSX file as the base + list for files/directories that need to be copied + which have the 'redistribute' flagged. + --with-structures Copy sources with directory structure. + --zip Zip the copied sources to the output location. + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Collect sources that have 'redistribute' flagged as 'True' in .ABOUT +files or inventory to the output location. + +Details +^^^^^^^ + + .. code-block:: + + --from-inventory + + Provide an inventory CSV/JSON file with the 'redistribute' field filled as + the indication of which files/sources need to be copied. + + $ about collect_redist_src --from-inventory 'path to the inventory' LOCATION OUTPUT + + --with-structures + + Copy the file(s) along with its parent directories + + For instance, assuming we want to copy the following file: + /project/work/hello/foo.c + + OUTPUT: /output/ + + $ about collect_redist_src --with-structure /project/ /output/ + + OUTPUT: /output/work/hello/foo.c + + $ about collect_redist_src /project/ /output/ + + OUTPUT: /output/foo.c + + --zip + + Zip the copied sources to the output location + + $ about collect_redist_src --zip /project/ /output/output.zip + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +gen +=== + +Syntax +------ + + .. code-block:: + + about gen [OPTIONS] LOCATION OUTPUT + + LOCATION: Path to a JSON/CSV/XLSX inventory file. + OUTPUT: Path to a directory where ABOUT files are generated. + +Options +------- + + .. code-block:: + + --android Generate MODULE_LICENSE_XXX (XXX will be + replaced by license key) and NOTICE as the + same design as from Android. + --fetch-license Fetch license data and text files from the + ScanCode LicenseDB. + --fetch-license-djc api_url api_key + Fetch license data and text files from a + DejaCode License Library API URL using the + API KEY. + --reference DIR Path to a directory with reference license + data and text files. + --worksheet name The worksheet name from the INPUT. (Default: + the "active" worksheet) + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location. + +Details +^^^^^^^ + + .. code-block:: + + --android + + Create an empty file named `MODULE_LICENSE_XXX` where `XXX` is the license + key and create a NOTICE file which these two files follow the design from + Android Open Source Project. + + The input **must** have the license key information as this is needed to + create the empty MODULE_LICENSE_XXX + + $ about gen --android LOCATION OUTPUT + + --fetch-license + + Fetch licenses text and create .LICENSE side-by-side + with the generated .ABOUT file using the data fetched from the the ScanCode LicenseDB. + + The input needs to have the 'license_expression' field. + + $ about gen --fetch-license LOCATION OUTPUT + + --fetch-license-djc + + Fetch licenses text from a DejaCode API, and create .LICENSE side-by-side + with the generated .ABOUT file using the data fetched from the DejaCode License Library. + + This option requires 2 parameters: + api_url - URL to the DJE License Library. + api_key - Hash key to authenticate yourself in the API. + + In addition, the input needs to have the 'license_expression' field. + (Please contact nexB to get the api_* value for this feature) + + $ about gen --fetch-license-djc 'api_url' 'api_key' LOCATION OUTPUT + + --reference + + Copy the reference files such as 'license_files' and 'notice_files' to the + generated location from the specified directory. + + For instance, + the specified directory, /home/licenses_notices/, contains all the licenses and notices: + /home/licenses_notices/apache2.LICENSE + /home/licenses_notices/jquery.js.NOTICE + + $ about gen --reference /home/licenses_notices/ LOCATION OUTPUT + + --worksheet + + This option identify the worksheet name from the XLSX input to work with. + If no worksheet is defined, the "active" worksheet will be used + + $ about gen --worksheet BOM LOCATION OUTPUT + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +Special Notes +------------- +If the input contains values for license_file, the tool will attempt to +associate the license_file with the corresponding license_key. + +sample.csv + ++----------------+------+---------------------+--------------+ +| about_resource | name | license_expression | license_file | ++================+======+=====================+==============+ +| /project/test.c| test.c | mit AND custom | custom.txt | ++----------------+------+---------------------+--------------+ + +If the user does not utilize the **--fetch-license** option, the input will +contain two license keys and one license file. In this scenario, the tool cannot +determine which license key the license file is referencing. As a result, the +license_file will be saved separately. + +i.e. + + .. code-block:: + + about_resource: test.c + name: test.c + license_expression: mit AND custom + licenses: + - key: mit + name: mit + - key: custom + name: custom + - file: custom.txt + +On the other hand, if the user generates ABOUT files using the +**--fetch-license** option, the MIT license will be retrieved. This will result +in having one license key and one license file. In such cases, the tool will +consider it a successful match. + +i.e. + + .. code-block:: + + about_resource: test.c + name: test.c + license_expression: mit AND custom + licenses: + - key: mit + name: MIT License + file: mit.LICENSE + url: https://scancode-licensedb.aboutcode.org/mit.LICENSE + spdx_license_key: MIT + - key: custom + name: custom + file: custom.txt + +gen_license +=========== + +Syntax +------ + + .. code-block:: + + about gen_license [OPTIONS] LOCATION OUTPUT + + LOCATION: Path to a JSON/CSV/XLSX/.ABOUT file(s) + OUTPUT: Path to a directory where license files are saved. + +Options +------- + + .. code-block:: + + --djc api_url api_key Fetch licenses from a DejaCode License Library. + --scancode Indicate the input JSON file is from + scancode_toolkit. + --worksheet name The worksheet name from the INPUT. (Default: the + "active" worksheet) + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Fetch licenses (Default: ScanCode LicenseDB) in the license_expression +field and save to the output location. + +Details +^^^^^^^ + + .. code-block:: + + --djc + + Fetch licenses text from a DejaCode API, and create .LICENSE to the + OUTPUT Location using the data fetched from the DejaCode License Library. + + This option requires 2 parameters: + api_url - URL to the DJE License Library. + api_key - Hash key to authenticate yourself in the API. + + In addition, the input needs to have the 'license_expression' field. + (Please contact nexB to get the api_* value for this feature) + + $ about gen_license --djc 'api_url' 'api_key' LOCATION OUTPUT + + --scancode + + Indicates the JSON input is from scancode toolkit license detection + + $ about gen_license --scancode /home/project/scancode-license-detection.json OUTPUT + + --worksheet + + This option identify the worksheet name from the XLSX input to work with. + If no worksheet is defined, the "active" worksheet will be used + + $ about gen_license --worksheet BOM /home/project/bom-v0.10.xlsx OUTPUT + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +Special Notes +------------- +If no `--djc` option is set, the tool will default to fetch licenses from ScanCode LicenseDB. + +inventory +========= + +Syntax +------ + + .. code-block:: + + about inventory [OPTIONS] LOCATION OUTPUT + + LOCATION: Path to an ABOUT file or a directory with ABOUT files. + OUTPUT: Path to the CSV/JSON/XLSX inventory file to create, or using '-' to + print result on screen/to stdout (Excel-formatted output cannot be used in + stdout). + +Options +------- + + .. code-block:: + + --exclude PATTERN Exclude the processing of the specified input pattern + (e.g. *tests* or test/). + -f, --format [json|csv|excel] Set OUTPUT file format. [default: csv] + -q, --quiet Do not print any error/warning. + --verbose Show all the errors and warning. + -h, --help Show this message and exit. + +Purpose +------- + +Create a JSON/CSV/XLSX inventory of components from ABOUT files, or +use `-` to print result to stdout. + +Details +^^^^^^^ + + .. code-block:: + + --exclude PATTERN + + Exclude the processing of the specified input pattern + + $ about inventory --exclude "tests*" LOCATION OUTPUT + + -f, --format [json|csv|excel] + + Set OUTPUT file format. [default: csv] + + $ about inventory -f json LOCATION OUTPUT + + $ about inventory -f json LOCATION - + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +Special Notes +------------- + +Multiple licenses support format +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The multiple licenses support format for CSV files are separated by line break + ++----------------+------+--------------+---------------+---------------------+ +| about_resource | name | license_key | license_name | license_file | ++================+======+==============+===============+=====================+ +| test.tar.xz | test | | apache-2.0 | | Apache 2.0 | | apache-2.0.LICENSE| +| | | | mit | | MIT License | | mit.LICENSE | ++----------------+------+--------------+---------------+---------------------+ + +The multiple licenses support format for ABOUT files are by "grouping" with the keyword "licenses" + + .. code-block:: + + about_resource: test.tar.xz + name: test + licenses: + - key: apache 2.0 + name: Apache 2.0 + file: apache-2.0.LICENSE + - key: mit + name: MIT License + file: mit.LICENSE + +Multiple license_file support +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To support multiple license file for a license, the correct format is to separate by comma + ++----------------+------+--------------+---------------+---------------------+ +| about_resource | name | license_key | license_name | license_file | ++================+======+==============+===============+=====================+ +| test.tar.xz | test | | gpl-2.0 | | GPL 2.0 | | COPYING, COPYINGv2| +| | | | mit | | MIT License | | mit.LICENSE | ++----------------+------+--------------+---------------+---------------------+ + + .. code-block:: + + about_resource: test.tar.xz + name: test + licenses: + - key: gpl-2.0 + name: gpl-2.0 + file: COPYING, COPYING.v2 + - key: mit + name: mit + file: mit.LICENSE + +Note that if license_name is not provided, the license key will be used as the license name. + +`--exclude` +^^^^^^^^^^^ + +As the `--exclude` option accepts patterns that may include wildcards, the running shell could +expand these patterns into filenames, potentially causing errors. To avoid this, +ensure the pattern is wrapped in quotes. + +On Windows, users can either use `^` to escape the `*` or use `--%` +before `--exclude` to prevent shell globbing. + +$ about check /home/project/about_files/ --exclude "tests^*" + +$ about check /home/project/about_files/ --% --exclude "*tests*" + +transform +========= + +Syntax +------ + + .. code-block:: + + about transform [OPTIONS] LOCATION OUTPUT + + LOCATION: Path to a CSV/JSON/XLSX file. + OUTPUT: Path to CSV/JSON/XLSX inventory file to create. + +Options +------- + + .. code-block:: + + -c, --configuration FILE Path to an optional YAML configuration file. See + --help-format for format help. + --worksheet name The worksheet name from the INPUT. (Default: the + "active" worksheet) + --help-format Show configuration file format help and exit. + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. + +Purpose +------- + +Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, +filters and checks and then write a new CSV/JSON/Excel to OUTPUT. + +Details +^^^^^^^ + + .. code-block:: + + -c, --configuration + + Path to an optional YAML configuration file. See--help-format for format help. + + $ about transform -c 'path to the YAML configuration file' LOCATION OUTPUT + + --worksheet + + This option identify the worksheet name from the XLSX input to work with. + If no worksheet is defined, the "active" worksheet will be used + + $ about transform -c 'path to the YAML configuration file' --worksheet BOM /project/bom-v.20.xlsx OUTPUT + + --help-format + + Show configuration file format help and exit. + This option will print out examples of the the YAML configuration file. + + Keys configuration are: `field_renamings`, `required_fields` and `field_filters` + + $ about transform --help-format + + --verbose + + This option tells the tool to show all errors found. + The default behavior will only show 'CRITICAL', 'ERROR', and 'WARNING' + +--help-format +------------- + + .. code-block:: + + A transform configuration file is used to describe which transformations and + validations to apply to a source CSV file. This is a simple text file using YAML + format, using the same format as an .ABOUT file. + + The attributes that can be set in a configuration file are: + + * field_renamings: + An optional map of source CSV or JSON field name to target CSV/JSON new field name that + is used to rename CSV fields. + + For instance with this configuration the fields "Directory/Location" will be + renamed to "about_resource" and "foo" to "bar": + field_renamings: + about_resource : 'Directory/Location' + bar : foo + + The renaming is always applied first before other transforms and checks. All + other field names referenced below are these that exist AFTER the renamings + have been applied to the existing field names. + + * required_fields: + An optional list of required field names that must have a value, beyond the + standard fields names. If a source CSV/JSON does not have such a field or a row is + missing a value for a required field, an error is reported. + + For instance with this configuration an error will be reported if the fields + "name" and "version" are missing or if any row does not have a value set for + these fields: + required_fields: + - name + - version + + * field_filters: + An optional list of field names that should be kept in the transformed CSV/JSON. If + this list is provided, all the fields from the source CSV/JSON that should be kept + in the target CSV/JSON must be listed regardless of either standard or required + fields. If this list is not provided, all source CSV/JSON fields are kept in the + transformed target CSV/JSON. + + For instance with this configuration the target CSV/JSON will only contains the "name" + and "version" fields and no other field: + field_filters: + - name + - version + + * exclude_fields: + An optional list of field names that should be excluded in the transformed CSV/JSON. If + this list is provided, all the fields from the source CSV/JSON that should be excluded + in the target CSV/JSON must be listed. Excluding standard or required fields will cause + an error. If this list is not provided, all source CSV/JSON fields are kept in the + transformed target CSV/JSON. + + For instance with this configuration the target CSV/JSON will not contain the "type" + and "temp" fields: + exclude_fields: + - type + - temp + +Example +------- + +fields renaming +^^^^^^^^^^^^^^^ + +conf.txt +"""""""" + + .. code-block:: + + field_renamings: + about_resource : 'Directory / Filename' + name : Component + version: 'Confirmed Version' + license_expression: 'Confirmed License Expression' + + +input.csv +""""""""" + ++----------------------+-----------+--------------------+------------------------------+ +| Directory / Filename | Component | Confirmed Version | Confirmed License Expression | ++======================+===========+====================+==============================+ +| /project/sample/ | sample | v 1.2.3 | apache-2.0 | ++----------------------+-----------+--------------------+------------------------------+ + + +Command +""""""" + + .. code-block:: + + about transform -c conf.txt input.csv output.csv + +The result output will look like the following: + +output.csv +"""""""""" + ++------------------+--------+---------+--------------------+ +| about_resource | name | version | license_expression | ++==================+========+=========+====================+ +| /project/sample/ | sample | v 1.2.3 | apache-2.0 | ++------------------+--------+---------+--------------------+ + +Special Notes +------------- +When using the field_filters configuration, all the standard required +columns (name) and the user defined required_fields +need to be included. + +Notes +===== +The AboutCode Toolkit version 10.0.0 will work with input from Scancode Toolkit +version 32.0.0 or later. If you are using an earlier version of Scancode Toolkit, +specifically version 31 or older, it will only be compatible with prior versions +of AboutCode Toolkit. + + +Configure proxy +--------------- +The `requests` library is used since AboutCode Toolkit version 10.1.0. To do the +http request, users can set the standard environment variables **http_proxy**, +**https_proxy**, **no_proxy**, **all_proxy** with the export statement + +i.e. + + .. code-block:: + + $ export HTTP_PROXY="http://10.10.1.10:3128" + $ export HTTPS_PROXY="http://10.10.1.10:1080" + $ export ALL_PROXY="socks5://10.10.1.10:3434" + +See https://requests.readthedocs.io/en/latest/user/advanced/#proxies for +references diff --git a/docs/source/specification.rst b/docs/source/specification.rst new file mode 100644 index 00000000..b6f500e5 --- /dev/null +++ b/docs/source/specification.rst @@ -0,0 +1,523 @@ +.. _specification: + +=============================== +ABOUT File Specification v4.0.1 +=============================== + +Purpose +======= + +An ABOUT file provides a simple way to document the provenance (origin and license) +and other important or interesting information about a software component. +An ABOUT file is a small YAML formatted text file stored in the codebase side-by-side +with the software component file or archive that it documents. No modification +of the documented software is needed. + +The ABOUT format is plain text with field name/value pairs separated by a colon. +It is easy to read and create by hand and is designed first for humans, rather than +machines. The format is well-defined and structured just enough to make it easy to process with +software as well. It contains enough information to fulfill key license requirements +such as creating credits or attribution notices, collecting redistributable source code, +or providing information about new versions of a software component. + +Getting Started +=============== + +A simple and valid ABOUT file named httpd-2.4.3.tar.gz.ABOUT may look like this: + + .. code-block:: + + about_resource: httpd-2.4.3.tar.gz + name: Apache HTTP Server + version: 2.4.3 + homepage_url: http://httpd.apache.org + download_url: http://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz + license_expression: apache-2.0 + licenses: + - key: apache-2.0 + name: Apache 2.0 + file: apache-2.0.LICENSE + url: https://scancode-licensedb.aboutcode.org/apache-2.0.LICENSE + spdx_license_key: Apache-2.0 + notice_file: httpd.NOTICE + copyright: Copyright (c) 2012 The Apache Software Foundation. + +The meaning of this ABOUT file is: + +- The file "httpd-2.4.3.tar.gz" is stored in the same directory and side-by-side with + the ABOUT file "httpd-2.4.3.tar.gz.ABOUT" that documents it. +- The name of this component is "Apache HTTP Server" with version "2.4.3". +- The homepage URL for this component is https://httpd.apache.org +- The file "httpd-2.4.3.tar.gz" was originally downloaded from + https://archive.apache.org/dist/httpd/httpd-2.4.3.tar.gz +- This component is licensed under "apache-2.0" +- The licenses section contains the information of this "apache-2.0" license. +- In the same directory, "apache-2.0.LICENSE" and "httpd.NOTICE" are files + that contain respectively the license text and the notice text for this component. + +Specification +============= + +An ABOUT file is an YAML formatted text file. +The key for the licenses field and the license_expression are ScanCode license key. + +ABOUT file name +--------------- + +An ABOUT file name can use a limited set of characters and is suffixed with a +".ABOUT" extension using any combination of uppercase and lowercase characters. + +A file name can contain any characters and digits with the following exception and condition: + +- the following symbols are not accepted: ``", #, &, ', *, \, :, ;, <, >, =, ?, /, ^, `, |`` +- The case of a file name is not significant. On case-sensitive file systems + (such as on Linux), a tool must report an error if two ABOUT files stored in the same + directory have the same lowercase file name. This is to ensure that ABOUT files can be + used across file systems. The convention is to use a lowercase file name and an uppercase + ABOUT extension. + +Lines of text +------------- + +An ABOUT file contains lines of text. Lines contain field names/values pairs. +The standard line ending is the LF character. The line ending characters can be any LF, +CR or CR/LF and tools must normalize line endings to LF when processing an ABOUT file. +Empty lines and lines containing only white spaces that are not part of a field value +continuation are ignored. Empty lines are commonly used to improve the readability +of an ABOUT file. + +Field name +---------- + +A field name can contain only these US-ASCII characters: + +- digits from 0 to 9 +- uppercase and lowercase letters from A to Z +- the ``"_"`` underscore sign. +- Field names are not case sensitive. For example, "HOMEPAGE_URL" and "HomePage_url" + represent the same field name. +- A field name must start at the beginning of a new line. No spaces is allowed in + the field name. It can be followed by one or more spaces that must be ignored. + These spaces are commonly used to improve the readability of an ABOUT file. + +Field value +----------- + +The field value is separated from the field name by a ":" colon. The ":" colon +can be followed by one or more spaces that must be ignored. This also applies +to trailing white spaces: they must be ignored. + +The field value is composed of one or more lines of plain printable text. + +When a field value is a long string, additional continuation lines must start with +at least one space. In this case, the first space of an additional continuation +line is ignored and should be removed from the field value by tools. + +For instance: + + .. code-block:: + + description: This is a long description for a + software component that additional continuation line is used. + +When a field value contains more than one line of text, a "literal block" (using ``|``) is need. + +For instance: + + .. code-block:: + + description: | + This is a long description for a software component that spans + multiple lines with arbitrary line breaks. + + This text contains multiple lines. + +Fields are mandatory, optional or custom extension +-------------------------------------------------- + +A field can be mandatory, optional or custom extension. Tools must +report an error for missing mandatory fields. + +Fields validation +----------------- + +When processing an ABOUT file, tools must report a warning or error if a field +is invalid. A field can be invalid for several reasons, such as invalid field +name syntax or invalid content. Tools should report additional validation error +details. The validation process should check that each field name is syntactically +correct and that fields contain correct values according to its concise, common +sense definition in this specification. For certain fields, additional and specific +validations are relevant such as URL validation, path resolution and verification, +and so forth. Tools should report a warning for present fields that do not have any value. + +Fields order and multiple occurrences +------------------------------------- + +The field order does not matter. Multiple occurrences of a field name is +not supported. + +The tool processing an ABOUT file or CSV/JSON/XLSX input will issue an error +when a field name occurs more than once in the input file. + +Field referencing a file +------------------------ + +The actual value of some fields may be contained in another file. This is useful +for long texts or to reference a common text in multiple ABOUT files such as a +common license text. In this case the field name is suffixed with "_file" and +the field value must be a path pointing to the file that contains the actual +value of the field. If the field is referencing a license file, a "file" field +within the "licenses" group can be used. This path must be a POSIX path relative +to the path of the ABOUT file. The file content must be UTF-8-encoded text. + +For example, this example shows the license file for the component is named +"linux.COPYING" and the notice file is "NOTICE": + + .. code-block:: + + license_file: linux.COPYING + notice_file: NOTICE + +Alternatvely, it can also write as the follow: + + .. code-block:: + + licenses: + - file: linux.COPYING + notice_file: NOTICE + +In this example, the README file is stored in a doc directory, one directory +above the ABOUT file directory, using a relative POSIX path: + + .. code-block:: + + licenses: + - file: ../docs/ruby.README + +In addition, there may be cases that a license can have 2 or more referenced +license files. If this is the case, a comma ',' is used to identify multiple +files For instance: + + .. code-block:: + + license_expression: gpl-2.0-plus + licenses: + - key: gpl-2.0-plus + file: COPYING, COPYING.LESSER + +Field referencing a URL +----------------------- + +The value of a field may reference URLs such as a homepage or a download. In this +case the field name is suffixed with "_url" and the field value must be a valid +absolute URL starting with ``ftp://``, ``http://`` or ``https://``. URLs are informational +and the content they may reference is ignored. For example, a download URL +is referenced this way: + + .. code-block:: + + download_url: http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.4.20.tar.bz2 + +Flag fields +----------- + +Flag fields have a "true" or "false" value. ``True``, ``T``, ``Yes``, +``Y`` or ``x`` must be interpreted as "true" in any case combination. +``False``, ``F``, ``No`` or ``N`` must be interpreted as "false" +in any case combination. + +Referencing the file or directory documented by an ABOUT file +------------------------------------------------------------- + +An ABOUT file documents one file or directory. The ``about_resource`` +field reference the documented file or directory. The value of the ``about_resource`` +field is the name or path of the referenced file or directory. There is also a +``ignored_resources`` field which can be used to ignore a set of subpaths inside the +directory which is being documented in the ABOUT file. + +By convention, an ABOUT file is often stored in the same directory side-by-side +to the file or directory that it documents, but this is not mandatory. + +For example, a file named django.ABOUT contains the following field to document +the django-1.2.3.tar.gz archive stored in the same directory: + + .. code-block:: + + about_resource: django-1.2.3.tar.gz + +In this example, the ABOUT file documents a whole sub-directory: + + .. code-block:: + + about_resource: linux-kernel-2.6.23 + +In this example, the ABOUT file documents a whole sub-directory, with some +sub-paths under the directory ignored: + + .. code-block:: + + about_resource: linux-kernel-2.6.23 + ignored_resources: linux-kernel-2.6.23/Documentation + +In this example, the ABOUT file documents the current directory, using a "." period to reference it: + + .. code-block:: + + about_resource: . + +Mandatory fields +---------------------- + +When a tool processes an ABOUT file, it must issue an error if the +mandatory field is missing. + +- name: Component name. + +Optional Information fields +--------------------------- + +- about_resource: The resource this file referencing to. +- ignored_resources: A list of paths under the ``about_resource`` path, which are + not documented in the ABOUT file, and the information in the ABOUT file does not + apply to these subpaths. +- version: Component or package version. A component or package usually has a version, + such as a revision number or hash from a version control system (for a snapshot checked + out from VCS such as Subversion or Git). If not available, the version should be the date + the component was provisioned, in an ISO date format such as 'YYYY-MM-DD'. +- spec_version: The version of the ABOUT file format specification used for this file. + This is provided as a hint to readers and tools in order to support future versions + of this specification. +- description: Component description, as a short text. +- download_url: A direct URL to download the original file or archive documented + by this ABOUT file. +- homepage_url: URL to the homepage for this component. +- changelog_file: Changelog file for the component. +- package_url: Package URL for the package. +- notes: Notes and comments about the component. + +Optional Owner and Author fields +-------------------------------- + +- owner: The name of the primary organization or person(s) that owns or + provides the component. +- owner_url: URL to the homepage for the owner. +- contact: Contact information (such as an email address or physical address) + for the component owner. +- author: Name of the organization(s) or person(s) that authored the component. +- author_file: Author file for the component. + +Optional Licensing fields +------------------------- + +- copyright: Copyright statement for the component. +- notice_file: Legal notice or credits for the component. +- notice_url: URL to a legal notice for the component. +- license_file: License file that applies to the component. For example, the + name of a license file such as LICENSE or COPYING file extracted from a + downloaded archive. +- license_url: URL to the license text for the component. +- license_expression: The ScanCode license expression that apply to + the component. You can separate each identifier using " or " and " and " to + document the relationship between multiple license identifiers, such as a choice + among multiple licenses (No special characters are allowed). +- license_name: The ScanCode license short name for the license + (No special characters are allowed). +- license_key: The ScanCode license key(s) for the component + (No special characters are allowed). +- spdx_license_key: The ScanCode LicenseDB spdx_license_key defined + for the license at https://scancode-licensedb.aboutcode.org/index.html +- spdx_license_expression: The license expression that use spdx_license_key + +Notes +^^^^^ +The license_* fields in the generated .ABOUT files are grouped under the "licenses" fields. +For instance, + + .. code-block:: + + licenses: + - key: apache-2.0 + name: Apache 2.0 + file: apache-2.0.LICENSE + url: https://scancode-licensedb.aboutcode.org/apache-2.0.LICENSE + spdx_license_key: Apache-2.0 + +However, if user create .ABOUT file manually, it can also used the individual field name. + + + .. code-block:: + + license_key: apache-2.0 + license_name: Apache 2.0 + license_file: apache-2.0.LICENSE + license_url: https://scancode-licensedb.aboutcode.org/apache-2.0.LICENSE + spdx_license_key: Apache-2.0 + +These groupping is only used in the generated .ABOUT files. The output from **gen** +will use the individual field name. + +Optional Boolean flag fields +---------------------------- + +- redistribute: Set this flag to yes if the component license requires source code + redistribution. Defaults to no when absent. +- track_changes: Set this flag to yes if the component license requires tracking changes made to + a the component. Defaults to no when absent. +- modified: Set this flag to yes if the component has been modified. Defaults to no when absent. +- internal_use_only: Set this flag to yes if the component is used internal only. + Defaults to no when absent. + +Optional Boolean and Character fields +------------------------------------- + +- attribute: This field can be either in boolean value: ('yes', 'y', 'true', + 'x', 'no', 'n', 'false') or a character value field with no more than 2 + characters. Defaults to no when absent. + +Optional Extension fields +------------------------- + +You can create extension fields by prefixing them with a short prefix to +distinguish these from the standard fields (but this is not necessary). + +Optional Extension fields to reference files stored in a version control system (VCS) +------------------------------------------------------------------------------------- +These fields provide a simple way to reference files stored in a version +control system. There are many VCS tools such as CVS, Subversion, Git, +ClearCase and GNU Arch. Accurate addressing of a file or directory revision +in each tool in a uniform way may not be possible. Some tools may require access +control via user/password or certificate and this information should not be +stored in an ABOUT file. This extension defines the 'vcs' field extension +prefix and a few common fields to handle the diversity of ways that VCS +tools reference files and directories under version control: + +- vcs_tool: VCS tool such as git, svn, cvs, etc. +- vcs_repository: Typically a URL or some other identifier used by a + VCS tool to point to a repository such as an SVN or Git repository URL. +- vcs_path: Path used by a particular VCS tool to point to a file, + directory or module inside a repository. +- vcs_tag: tag name or path used by a particular VCS tool. +- vcs_branch: branch name or path used by a particular VCS tool. +- vcs_revision: revision identifier such as a revision hash or version number. + +Some examples for using the vcs_* extension fields include: + + .. code-block:: + + vcs_tool: svn + vcs_repository: http://svn.code.sf.net/p/inkscape/code/inkscape_project/ + vcs_path: trunk/inkscape_planet/ + vcs_revision: 22886 + +or: + + .. code-block:: + + vcs_tool: git + vcs_repository: git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git + vcs_path: tools/lib/traceevent + vcs_revision: b59958d90b3e75a3b66cd311661535f94f5be4d1 + +Optional Extension fields for checksums +--------------------------------------- +These fields support checksums (such as SHA1 and MD5)commonly provided with +downloaded archives to verify their integrity. A tool can optionally use these +to verify the integrity of a file documented by an ABOUT file. + +- checksum_md5: MD5 for the file documented by this ABOUT file in the "about_resource" field. +- checksum_sha1: SHA1 for the file documented by this ABOUT file in the "about_resource" field. +- checksum_sha256: SHA256 for the file documented by this ABOUT file in + the "about_resource" field. + +Some examples: + + .. code-block:: + + checksum_md5: f30b9c173b1f19cf42ffa44f78e4b96c + +Fields Type +----------- + +Following are the types for the supporting fields (All the custom fields +will be treated as **StringField**): + +.. list-table:: + :widths: 10 10 + :header-rows: 1 + + * - Type + - Fields + * - AboutResourceField + - | about_resource + | ignored_resources + * - BooleanField + - | redistribute + | track_changes + | modified + | internal_use_only + * - BooleanAndTwoCharactersField + - attribute + * - FileTextField + - | license_file + | notice_file + | changelog_file + | author_file + * - ListField + - | license_key + | license_name + | spdx_license_key + * - PackageUrlField + - package_url + * - SingleLineField + - | name + | version + | license_expression + | spdx_license_expression + | declared_license_expression + | other_license_expression + | vcs_tool + | vcs_repository + | vcs_path + | vcs_tag + | vcs_branch + | vcs_revision + | checksum_md5 + | checksum_sha1 + | checksum_sha256 + | spec_version + * - StringField + - | description + | notes + | copyright + | owner + | contact + | author + * - UrlField + - | download_url + | homepage_url + | notice_url + | owner_url + * - UrlListField + - license_url + +Type description +---------------- + +- **AboutResourceField**: Path or list of path to the about resource. +- **BooleanField**: An flag field with a boolean value. Validated value is False, + True or None. +- **BooleanAndTwoCharactersField**: Field with either a boolean value or + character(s) value (at most 2 characters). Validated value is False, True, + None or character value. +- **FileTextField**: A path field pointing to one or more text files such as + license files. The validated value is an ordered dict of path->Text or None + if no location or text could not be loaded. +- **ListField**: A field containing a list of string values, one per line. The + validated value is a list. +- **PackageUrlField**: A Package URL field. The validated value is a purl +- **SingleLineField**: A field containing a string value on a single line. The + validated value is a string. +- **StringField**: A field containing a string value possibly on multiple lines. + The validated value is a string. +- **UrlField**: A URL field. The validated value is a URL. +- **UrlListField**: A URL field. The validated value is a list of URLs. diff --git a/docs/source/type_of_errors.rst b/docs/source/type_of_errors.rst new file mode 100644 index 00000000..fbbcd9a6 --- /dev/null +++ b/docs/source/type_of_errors.rst @@ -0,0 +1,93 @@ +.. _type_of_errors: + +============== +Type of Errors +============== + +We have 6 type of errors as describe below: + +NOTSET +====== + +Trigger: +-------- + + * None + +Details +^^^^^^^ + + We do not have event to trigger this error. + + +DEBUG +===== + +Trigger: +-------- + + * None + +Details +^^^^^^^ + + We do not have event to trigger this error. + + +INFO +==== + +Trigger: +-------- + + * `about_resource` not found + * Custom fields detected + * Empty field value + + +WARNING +======= + +Trigger: +-------- + + * Duplicated value being ignored + * Invalid Package URL from input + * Invalid URL from input + + +ERROR +===== + +Trigger: +-------- + + * Invalid license + * Invalid API call + * Invalid character + * Invalid input + * Duplicated field name + * Incorrect input format + * Failure to write ABOUT file + * Network problem + + +CRITICAL +======== + +Trigger: +-------- + + * Invalid template + * File field not found + * Duplicated `about_resource` + * Not supported field format + * Essential or required field not found + * Internal error + * Empty ABOUT file + * Invalid ABOUT file + + +.. note:: + If `--verbose` is set, all the detected errors will be reported. + Otherwise, only "CRITICAL", "ERROR" and 'WARNING" will be reported. diff --git a/etc/ci/azure-container-deb.yml b/etc/ci/azure-container-deb.yml new file mode 100644 index 00000000..d80e8dfb --- /dev/null +++ b/etc/ci/azure-container-deb.yml @@ -0,0 +1,50 @@ +parameters: + job_name: '' + container: '' + python_path: '' + python_version: '' + package_manager: apt-get + install_python: '' + install_packages: | + set -e -x + sudo apt-get -y update + sudo apt-get -y install \ + build-essential \ + xz-utils zlib1g bzip2 libbz2-1.0 tar \ + sqlite3 libxml2-dev libxslt1-dev \ + software-properties-common openssl + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: 'ubuntu-22.04' + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-container-rpm.yml b/etc/ci/azure-container-rpm.yml new file mode 100644 index 00000000..a64138c9 --- /dev/null +++ b/etc/ci/azure-container-rpm.yml @@ -0,0 +1,51 @@ +parameters: + job_name: '' + image_name: 'ubuntu-22.04' + container: '' + python_path: '' + python_version: '' + package_manager: yum + install_python: '' + install_packages: | + set -e -x + sudo yum groupinstall -y "Development Tools" + sudo yum install -y \ + openssl openssl-devel \ + sqlite-devel zlib-devel xz-devel bzip2-devel \ + bzip2 tar unzip zip \ + libxml2-devel libxslt-devel + test_suite: '' + test_suite_label: '' + + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + container: + image: ${{ parameters.container }} + options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' + + steps: + - checkout: self + fetchDepth: 10 + + - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} + displayName: Install sudo + + - script: ${{ parameters.install_packages }} + displayName: Install required packages + + - script: ${{ parameters.install_python }} + displayName: 'Install Python ${{ parameters.python_version }}' + + - script: ${{ parameters.python_path }} --version + displayName: 'Show Python version' + + - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev + displayName: 'Run Configure' + + - script: ${{ parameters.test_suite }} + displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml new file mode 100644 index 00000000..9fdc7f15 --- /dev/null +++ b/etc/ci/azure-posix.yml @@ -0,0 +1,39 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each tsuite in parameters.test_suites }}: + ${{ tsuite.key }}: + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' + + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml new file mode 100644 index 00000000..26b41116 --- /dev/null +++ b/etc/ci/azure-win.yml @@ -0,0 +1,39 @@ +parameters: + job_name: '' + image_name: '' + python_versions: [] + test_suites: {} + python_architecture: x64 + +jobs: + - job: ${{ parameters.job_name }} + + pool: + vmImage: ${{ parameters.image_name }} + + strategy: + matrix: + ${{ each tsuite in parameters.test_suites }}: + ${{ tsuite.key }}: + test_suite_label: ${{ tsuite.key }} + test_suite: ${{ tsuite.value }} + + steps: + - checkout: self + fetchDepth: 10 + + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' + + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' + + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/install_sudo.sh b/etc/ci/install_sudo.sh new file mode 100644 index 00000000..77f4210d --- /dev/null +++ b/etc/ci/install_sudo.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + + +if [[ "$1" == "apt-get" ]]; then + apt-get update -y + apt-get -o DPkg::Options::="--force-confold" install -y sudo + +elif [[ "$1" == "yum" ]]; then + yum install -y sudo + +elif [[ "$1" == "dnf" ]]; then + dnf install -y sudo + +fi diff --git a/etc/ci/macports-ci b/etc/ci/macports-ci new file mode 100644 index 00000000..ac474e4e --- /dev/null +++ b/etc/ci/macports-ci @@ -0,0 +1,304 @@ +#! /bin/bash + +# Copyright (c) 2019 Giovanni Bussi + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +export COLUMNS=80 + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "COLUMNS=$COLUMNS" >> "$GITHUB_ENV" +fi + +# file to be source at the end of subshell: +export MACPORTS_CI_SOURCEME="$(mktemp)" + +( +# start subshell +# this allows to use the script in two ways: +# 1. as ./macports-ci +# 2. as source ./macports-ci +# as of now, choice 2 only changes the env var COLUMNS. + +MACPORTS_VERSION=2.6.4 +MACPORTS_PREFIX=/opt/local +MACPORTS_SYNC=tarball + +action=$1 +shift + +case "$action" in +(install) + +echo "macports-ci: install" + +KEEP_BREW=yes + +for opt +do + case "$opt" in + (--source) SOURCE=yes ;; + (--binary) SOURCE=no ;; + (--keep-brew) KEEP_BREW=yes ;; + (--remove-brew) KEEP_BREW=no ;; + (--version=*) MACPORTS_VERSION="${opt#--version=}" ;; + (--prefix=*) MACPORTS_PREFIX="${opt#--prefix=}" ;; + (--sync=*) MACPORTS_SYNC="${opt#--sync=}" ;; + (*) echo "macports-ci: unknown option $opt" + exit 1 ;; + esac +done + +if test "$KEEP_BREW" = no ; then + echo "macports-ci: removing homebrew" + pushd "$(mktemp -d)" + curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > uninstall + chmod +x uninstall + ./uninstall --force + popd +else + echo "macports-ci: keeping HomeBrew" +fi + +echo "macports-ci: prefix=$MACPORTS_PREFIX" + +if test "$MACPORTS_PREFIX" != /opt/local ; then + echo "macports-ci: Installing on non standard prefix $MACPORTS_PREFIX can be only made from sources" + SOURCE=yes +fi + +if test "$SOURCE" = yes ; then + echo "macports-ci: Installing from source" +else + echo "macports-ci: Installing from binary" +fi + +echo "macports-ci: Sync mode=$MACPORTS_SYNC" + +pushd "$(mktemp -d)" + +OSX_VERSION="$(sw_vers -productVersion | grep -o '^[0-9][0-9]*\.[0-9][0-9]*')" + +if test "$OSX_VERSION" == 10.10 ; then + OSX_NAME=Yosemite +elif test "$OSX_VERSION" == 10.11 ; then + OSX_NAME=ElCapitan +elif test "$OSX_VERSION" == 10.12 ; then + OSX_NAME=Sierra +elif test "$OSX_VERSION" == 10.13 ; then + OSX_NAME=HighSierra +elif test "$OSX_VERSION" == 10.14 ; then + OSX_NAME=Mojave +elif test "$OSX_VERSION" == 10.15 ; then + OSX_NAME=Catalina +else + echo "macports-ci: Unknown OSX version $OSX_VERSION" + exit 1 +fi + +echo "macports-ci: OSX version $OSX_VERSION $OSX_NAME" + +MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg + +# this is a workaround needed because binary installer MacPorts-2.6.3-10.12-Sierra.pkg is broken +if [ "$SOURCE" != yes ] && [ "$MACPORTS_PKG" = "MacPorts-2.6.3-10.12-Sierra.pkg" ] ; then + echo "macports-ci: WARNING $MACPORTS_PKG installer is broken" + echo "macports-ci: reverting to 2.6.2 installer followed by selfupdate" + MACPORTS_VERSION=2.6.2 + MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg +fi + +URL="https://distfiles.macports.org/MacPorts" +URL="https://github.com/macports/macports-base/releases/download/v$MACPORTS_VERSION/" + +echo "macports-ci: Base URL is $URL" + +if test "$SOURCE" = yes ; then +# download source: + curl -LO $URL/MacPorts-${MACPORTS_VERSION}.tar.bz2 + tar xjf MacPorts-${MACPORTS_VERSION}.tar.bz2 + cd MacPorts-${MACPORTS_VERSION} +# install + ./configure --prefix="$MACPORTS_PREFIX" --with-applications-dir="$MACPORTS_PREFIX/Applications" >/dev/null && + sudo make install >/dev/null +else + +# download installer: + curl -LO $URL/$MACPORTS_PKG +# install: + sudo installer -verbose -pkg $MACPORTS_PKG -target / +fi + +# update: +export PATH="$MACPORTS_PREFIX/bin:$PATH" + +echo "PATH=\"$MACPORTS_PREFIX/bin:\$PATH\"" > "$MACPORTS_CI_SOURCEME" + +if [ "$GITHUB_ACTIONS" = true ] ; then + echo "$MACPORTS_PREFIX/bin" >> "$GITHUB_PATH" +fi + + +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +case "$MACPORTS_SYNC" in +(rsync) + echo "macports-ci: Using rsync" + ;; +(github) + echo "macports-ci: Using github" + pushd "$MACPORTS_PREFIX"/var/macports/sources + sudo mkdir -p github.com/macports/macports-ports/ + sudo chown -R $USER:admin github.com + git clone https://github.com/macports/macports-ports.git github.com/macports/macports-ports/ + awk '{if($NF=="[default]") print "file:///opt/local/var/macports/sources/github.com/macports/macports-ports/"; else print}' "$SOURCES" > $HOME/$$.tmp + sudo mv -f $HOME/$$.tmp "$SOURCES" + popd + ;; +(tarball) + echo "macports-ci: Using tarball" + awk '{if($NF=="[default]") print "https://distfiles.macports.org/ports.tar.gz [default]"; else print}' "$SOURCES" > $$.tmp + sudo mv -f $$.tmp "$SOURCES" + ;; +(*) + echo "macports-ci: Unknown sync mode $MACPORTS_SYNC" + ;; +esac + +i=1 +# run through a while to retry upon failure +while true +do + echo "macports-ci: Trying to selfupdate (iteration $i)" +# here I test for the presence of a known portfile +# this check confirms that ports were installed +# notice that port -N selfupdate && break is not sufficient as a test +# (sometime it returns a success even though ports have not been installed) +# for some misterious reasons, running without "-d" does not work in some case + sudo port -d -N selfupdate 2>&1 | grep -v DEBUG | awk '{if($1!="x")print}' + port info xdrfile > /dev/null && break || true + sleep 5 + i=$((i+1)) + if ((i>20)) ; then + echo "macports-ci: Failed after $i iterations" + exit 1 + fi +done + +echo "macports-ci: Selfupdate successful after $i iterations" + +dir="$PWD" +popd +sudo rm -fr $dir + +;; + +(localports) + +echo "macports-ci: localports" + +for opt +do + case "$opt" in + (*) ports="$opt" ;; + esac +done + +if ! test -d "$ports" ; then + echo "macports-ci: Please provide a port directory" + exit 1 +fi + +w=$(which port) + +MACPORTS_PREFIX="${w%/bin/port}" + +cd "$ports" + +ports="$(pwd)" + +echo "macports-ci: Portdir fullpath: $ports" +SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf + +awk -v repo="file://$ports" '{if($NF=="[default]") print repo; print}' "$SOURCES" > $$.tmp +sudo mv -f $$.tmp "$SOURCES" + +portindex + +;; + +(ccache) +w=$(which port) +MACPORTS_PREFIX="${w%/bin/port}" + +echo "macports-ci: ccache" + +ccache_do=install + +for opt +do + case "$opt" in + (--save) ccache_do=save ;; + (--install) ccache_do=install ;; + (*) echo "macports-ci: ccache: unknown option $opt" + exit 1 ;; + esac +done + + +case "$ccache_do" in +(install) +# first install ccache +sudo port -N install ccache +# then tell macports to use it +CONF="${MACPORTS_PREFIX}"/etc/macports/macports.conf +awk '{if(match($0,"configureccache")) print "configureccache yes" ; else print }' "$CONF" > $$.tmp +sudo mv -f $$.tmp "$CONF" + +# notice that cache size is set to 512Mb, same as it is set by Travis-CI on linux +# might be changed in the future +test -f "$HOME"/.macports-ci-ccache/ccache.conf && + sudo rm -fr "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo mkdir -p "$MACPORTS_PREFIX"/var/macports/build/.ccache && + sudo cp -a "$HOME"/.macports-ci-ccache/* "$MACPORTS_PREFIX"/var/macports/build/.ccache/ && + sudo echo "max_size = 512M" > "$MACPORTS_PREFIX"/var/macports/build/.ccache/ccache.conf && + sudo chown -R macports:admin "$MACPORTS_PREFIX"/var/macports/build/.ccache + +;; +(save) + +sudo rm -fr "$HOME"/.macports-ci-ccache +sudo mkdir -p "$HOME"/.macports-ci-ccache +sudo cp -a "$MACPORTS_PREFIX"/var/macports/build/.ccache/* "$HOME"/.macports-ci-ccache/ + +esac + +CCACHE_DIR="$MACPORTS_PREFIX"/var/macports/build/.ccache/ ccache -s + +;; + +(*) +echo "macports-ci: unknown action $action" + +esac + +) + +# allows setting env var if necessary: +source "$MACPORTS_CI_SOURCEME" diff --git a/etc/ci/macports-ci.ABOUT b/etc/ci/macports-ci.ABOUT new file mode 100644 index 00000000..60a11f8e --- /dev/null +++ b/etc/ci/macports-ci.ABOUT @@ -0,0 +1,16 @@ +about_resource: macports-ci +name: macports-ci +version: c9676e67351a3a519e37437e196cd0ee9c2180b8 +download_url: https://raw.githubusercontent.com/GiovanniBussi/macports-ci/c9676e67351a3a519e37437e196cd0ee9c2180b8/macports-ci +description: Simplify MacPorts setup on Travis-CI +homepage_url: https://github.com/GiovanniBussi/macports-ci +license_expression: mit +copyright: Copyright (c) Giovanni Bussi +attribute: yes +checksum_md5: 5d31d479132502f80acdaed78bed9e23 +checksum_sha1: 74b15643bd1a528d91b4a7c2169c6fc656f549c2 +package_url: pkg:github/giovannibussi/macports-ci@c9676e67351a3a519e37437e196cd0ee9c2180b8#macports-ci +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/thirdparty/mit.LICENSE b/etc/ci/mit.LICENSE similarity index 99% rename from thirdparty/mit.LICENSE rename to etc/ci/mit.LICENSE index 14da7927..e662c786 100644 --- a/thirdparty/mit.LICENSE +++ b/etc/ci/mit.LICENSE @@ -1,5 +1,5 @@ -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/etc/conf/base.txt b/etc/conf/base.txt deleted file mode 100644 index b6fe4c05..00000000 --- a/etc/conf/base.txt +++ /dev/null @@ -1,24 +0,0 @@ -# Base configuration tools -certifi -setuptools -wheel -pip -wincertstore - -# used for templating -jinja2>=2.9.6 -MarkupSafe>=1.0.0 - -click==6.7 - -# used by schematics -six - -PyYAML >= 3.0, < 4.0 -attrs - -# license expression support -license_expression -boolean.py - --e . \ No newline at end of file diff --git a/etc/conf/dev/base.txt b/etc/conf/dev/base.txt deleted file mode 100644 index 65a6531a..00000000 --- a/etc/conf/dev/base.txt +++ /dev/null @@ -1,6 +0,0 @@ -# for tests -pytest -colorama -py -mock - diff --git a/etc/conf/win.txt b/etc/conf/win.txt deleted file mode 100644 index d73c1744..00000000 --- a/etc/conf/win.txt +++ /dev/null @@ -1 +0,0 @@ -wincertstore==0.2 diff --git a/etc/configure.py b/etc/configure.py deleted file mode 100644 index 253f7e30..00000000 --- a/etc/configure.py +++ /dev/null @@ -1,415 +0,0 @@ -#!/usr/bin/python - -# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. - -""" -This script is a configuration helper to select pip requirement files to install -and python and shell configuration scripts to execute based on provided config -directories paths arguments and the operating system platform. To use, create -a configuration directory tree that contains any of these: - - * Requirements files named with this convention: - - base.txt contains common requirements installed on all platforms. - - win.txt, linux.txt, mac.txt, posix.txt are os-specific requirements. - - * Python scripts files named with this convention: - - base.py is a common script executed on all os before os-specific scripts. - - win.py, linux.py, mac.py, posix.py are os-specific scripts to execute. - - * Shell or Windows CMD scripts files named with this convention: - - win.bat is a windows bat file to execute - - posix.sh, linux.sh, mac.sh are os-specific scripts to execute. - -The config directory structure contains one or more directories paths. This -way you can have a main configuration (that is always used) and additional -sub-configurations of a product such as for prod, test, ci, dev, or anything -else. - -All scripts and requirements are optional and only used if presents. Scripts -are executed in sequence, one after the other after all requirements are -installed, so they may import from any installed requirement. - -The execution order is: - - requirements installation - - python scripts execution - - shell scripts execution - -On posix, posix Python and shell scripts are executed before mac or linux -scripts. - -The base scripts or packages are always installed first before platform- -specific ones. - -For example a tree could be looking like this:: - etc/conf - base.txt : base pip requirements for all platforms - linux.txt : linux-only pip requirements - base.py : base config script for all platforms - win.py : windows-only config script - posix.sh: posix-only shell script - - etc/conf/prod - base.txt : base pip requirements for all platforms - linux.txt : linux-only pip requirements - linux.sh : linux-only script - base.py : base config script for all platforms - mac.py : mac-only config script -""" - -from __future__ import print_function - -import os -import stat -import sys -import shutil -import subprocess - - -# platform-specific file base names -sys_platform = str(sys.platform).lower() -on_win = False -if sys_platform.startswith('linux'): - platform_names = ('posix', 'linux',) -elif 'win32' in sys_platform: - platform_names = ('win',) - on_win = True -elif 'darwin' in sys_platform: - platform_names = ('posix', 'mac',) -else: - raise Exception('Unsupported OS/platform %r' % sys_platform) - platform_names = tuple() - - -# common file basenames for requirements and scripts -base = ('base',) - -# known full file names with txt extension for requirements -# base is always last -requirements = tuple(p + '.txt' for p in platform_names + base) - -# known full file names with py extensions for scripts -# base is always last -python_scripts = tuple(p + '.py' for p in platform_names + base) - -# known full file names of shell scripts -# there is no base for scripts: they cannot work cross OS (cmd vs. sh) -shell_scripts = tuple(p + '.sh' for p in platform_names) -if on_win: - shell_scripts = ('win.bat',) - - -def call(cmd, root_dir): - """ Run a `cmd` command (as a list of args) with all env vars.""" - cmd = ' '.join(cmd) - if subprocess.Popen(cmd, shell=True, env=dict(os.environ), cwd=root_dir).wait() != 0: - print() - print('Failed to execute command:\n%(cmd)s. Aborting...' % locals()) - sys.exit(1) - - -def find_pycache(root_dir): - """ - Yield __pycache__ directory paths found in root_dir as paths relative to - root_dir. - """ - for top, dirs, _files in os.walk(root_dir): - for d in dirs: - if d == '__pycache__': - dir_path = os.path.join(top, d) - dir_path = dir_path.replace(root_dir, '', 1) - dir_path = dir_path.strip(os.path.sep) - yield dir_path - - -def clean(root_dir): - """ - Remove cleanable directories and files in root_dir. - """ - print('* Cleaning ...') - cleanable = '''build bin lib Lib include Include Scripts local - django_background_task.log - develop-eggs eggs parts .installed.cfg - .Python - .cache - pip-selfcheck.json - '''.split() - - # also clean __pycache__ if any - cleanable.extend(find_pycache(root_dir)) - - for d in cleanable: - try: - loc = os.path.join(root_dir, d) - if os.path.exists(loc): - if os.path.isdir(loc): - shutil.rmtree(loc) - else: - os.remove(loc) - except: - pass - - -def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='): - """ - Return an iterable of pip command line options for `option` of pip using a - list of `paths` to directories. - """ - for path in paths: - if not os.path.isabs(path): - path = os.path.join(root_dir, path) - if os.path.exists(path): - yield option + '"' + path + '"' - - -def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False): - """ - Create a virtualenv in `root_dir` using the `std_python` Python - executable. One of the `tpp_dirs` must contain a vendored virtualenv.py and - virtualenv dependencies such as setuptools and pip packages. - - @std_python: Path or name of the Python executable to use. - - @root_dir: directory in which the virtualenv will be created. This is also - the root directory for the project and the base directory for vendored - components directory paths. - - @tpp_dirs: list of directory paths relative to `root_dir` containing - vendored Python distributions that pip will use to find required - components. - """ - if not quiet: - print("* Configuring Python ...") - # search virtualenv.py in the tpp_dirs. keep the first found - venv_py = None - for tpd in tpp_dirs: - venv = os.path.join(root_dir, tpd, 'virtualenv.py') - if os.path.exists(venv): - venv_py = '"' + venv + '"' - break - - # error out if venv_py not found - if not venv_py: - print("Configuration Error ... aborting.") - exit(1) - - vcmd = [std_python, venv_py, '--never-download'] - if quiet: - vcmd += ['--quiet'] - # third parties may be in more than one directory - vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir)) - # we create the virtualenv in the root_dir - vcmd.append('"' + root_dir + '"') - call(vcmd, root_dir) - - -def activate(root_dir): - """ Activate a virtualenv in the current process.""" - bin_dir = os.path.join(root_dir, 'bin') - activate_this = os.path.join(bin_dir, 'activate_this.py') - with open(activate_this) as f: - code = compile(f.read(), activate_this, 'exec') - exec(code, dict(__file__=activate_this)) - - -def install_3pp(configs, root_dir, tpp_dirs, quiet=False): - """ - Install requirements from requirement files found in `configs` with pip, - using the vendored components in `tpp_dirs`. - """ - if not quiet: - print("* Installing components ...") - requirement_files = get_conf_files(configs, root_dir, requirements, quiet) - if on_win: - bin_dir = os.path.join(root_dir, 'bin') - configured_python = os.path.join(bin_dir, 'python.exe') - base_cmd = [configured_python, '-m', 'pip'] - else: - base_cmd = ['pip'] - for req_file in requirement_files: - pcmd = base_cmd + ['install', '--upgrade', '--no-index', '--no-cache-dir'] - if quiet: - pcmd += ['--quiet'] - pip_dir_args = list(build_pip_dirs_args(tpp_dirs, root_dir, '--find-links=')) - pcmd.extend(pip_dir_args) - req_loc = os.path.join(root_dir, req_file) - pcmd.extend(['-r' , '"' + req_loc + '"']) - call(pcmd, root_dir) - - -def run_scripts(configs, root_dir, configured_python, quiet=False): - """ - Run Python scripts and shell scripts found in `configs`. - """ - if not quiet: - print("* Configuring ...") - # Run Python scripts for each configurations - for py_script in get_conf_files(configs, root_dir, python_scripts): - cmd = [configured_python, '"' + os.path.join(root_dir, py_script) + '"'] - call(cmd, root_dir) - - # Run sh_script scripts for each configurations - for sh_script in get_conf_files(configs, root_dir, shell_scripts): - if on_win: - cmd = [] - else: - # we source the scripts on posix - cmd = ['.'] - cmd.extend([os.path.join(root_dir, sh_script)]) - call(cmd, root_dir) - - -def chmod_bin(directory): - """ - Makes the directory and its children executable recursively. - """ - rwx = (stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR - | stat.S_IXGRP | stat.S_IXOTH) - for path, _, files in os.walk(directory): - for f in files: - os.chmod(os.path.join(path, f), rwx) - - -def get_conf_files(config_dir_paths, root_dir, file_names=requirements, quiet=False): - """ - Return a list of collected path-prefixed file paths matching names in a - file_names tuple, based on config_dir_paths, root_dir and the types of - file_names requested. Returned paths are posix paths. - - @config_dir_paths: Each config_dir_path is a relative from the project - root to a config dir. This script should always be called from the project - root dir. - - @root_dir: The project absolute root dir. - - @file_names: get requirements, python or shell files based on list of - supported file names provided as a tuple of supported file_names. - - Scripts or requirements are optional and only used if presents. Unknown - scripts or requirements file_names are ignored (but they could be used - indirectly by known requirements with -r requirements inclusion, or - scripts with python imports.) - - Since Python scripts are executed after requirements are installed they - can import from any requirement-installed component such as Fabric. - """ - # collect files for each requested dir path - collected = [] - for config_dir_path in config_dir_paths: - abs_config_dir_path = os.path.join(root_dir, config_dir_path) - if not os.path.exists(abs_config_dir_path): - if not quiet: - print('Configuration directory %(config_dir_path)s ' - 'does not exists. Skipping.' % locals()) - continue - # Support args like enterprise or enterprise/dev - paths = config_dir_path.strip('/').replace('\\', '/').split('/') - # a tuple of (relative path, location,) - current = None - for path in paths: - if not current: - current = (path, os.path.join(root_dir, path),) - else: - base_path, base_loc = current - current = (os.path.join(base_path, path), - os.path.join(base_loc, path),) - path, loc = current - # we iterate on known filenames to ensure the defined precedence - # is respected (posix over mac, linux), etc - for n in file_names: - for f in os.listdir(loc): - if f == n: - f_loc = os.path.join(loc, f) - if f_loc not in collected: - collected.append(f_loc) - - return collected - - -usage = '\nUsage: configure [--clean] ...\n' - - -if __name__ == '__main__': - - # you must create a CONFIGURE_QUIET env var if you want to run quietly - quiet = 'CONFIGURE_QUIET' in os.environ - - # define/setup common directories - etc_dir = os.path.abspath(os.path.dirname(__file__)) - root_dir = os.path.dirname(etc_dir) - - args = sys.argv[1:] - if args: - arg0 = args[0] - if arg0 == '--clean': - clean(root_dir) - sys.exit(0) - elif arg0.startswith('-'): - print() - print('ERROR: unknown option: %(arg0)s' % locals()) - print(usage) - sys.exit(1) - - sys.path.insert(0, root_dir) - bin_dir = os.path.join(root_dir, 'bin') - standard_python = sys.executable - - if on_win: - configured_python = os.path.join(bin_dir, 'python.exe') - scripts_dir = os.path.join(root_dir, 'Scripts') - bin_dir = os.path.join(root_dir, 'bin') - if not os.path.exists(scripts_dir): - os.makedirs(scripts_dir) - if not os.path.exists(bin_dir): - cmd = ('mklink /J "%(bin_dir)s" "%(scripts_dir)s"' % locals()).split() - call(cmd, root_dir) - else: - configured_python = os.path.join(bin_dir, 'python') - scripts_dir = bin_dir - - # Get requested configuration paths to collect components and scripts later - configs = [] - for path in args[:]: - abs_path = path - if not os.path.isabs(path): - abs_path = os.path.join(root_dir, path) - if not os.path.exists(abs_path): - print() - print('ERROR: Configuration directory does not exists:\n' - ' %(path)s: %(abs_path)r' - % locals()) - print(usage) - sys.exit(1) - - configs.append(path) - - # Collect vendor directories from environment variables: one or more third- - # party directories may exist as environment variables prefixed with TPP_DIR - thirdparty_dirs = [] - for envvar, path in os.environ.items(): - if not envvar.startswith('TPP_DIR'): - continue - abs_path = path - if not os.path.isabs(path): - abs_path = os.path.join(root_dir, path) - if not os.path.exists(abs_path): - if not quiet: - print() - print('WARNING: Third-party Python libraries directory does not exists:\n' - ' %(path)r: %(abs_path)r\n' - ' Provided by environment variable:\n' - ' set %(envvar)s=%(path)r' % locals()) - print() - else: - thirdparty_dirs.append(path) - - # Finally execute our three steps: venv, install and scripts - if not os.path.exists(configured_python): - create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=quiet) - activate(root_dir) - - install_3pp(configs, root_dir, thirdparty_dirs, quiet=quiet) - run_scripts(configs, root_dir, configured_python, quiet=quiet) - chmod_bin(bin_dir) - if not quiet: - print("* Configuration completed.") - print() diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst new file mode 100644 index 00000000..5e54a2cc --- /dev/null +++ b/etc/scripts/README.rst @@ -0,0 +1,112 @@ +This directory contains the tools to manage a directory of thirdparty Python +package source, wheels and metadata pin, build, update, document and publish to +a PyPI-like repo (GitHub release). + +NOTE: These are tested to run ONLY on Linux. + + +Thirdparty packages management scripts +====================================== + +Pre-requisites +-------------- + +* There are two run "modes": + + * To generate or update pip requirement files, you need to start with a clean + virtualenv as instructed below (This is to avoid injecting requirements + specific to the tools used here in the main requirements). + + * For other usages, the tools here can run either in their own isolated + virtualenv or in the the main configured development virtualenv. + These requireements need to be installed:: + + pip install --requirement etc/scripts/requirements.txt + +TODO: we need to pin the versions of these tools + + + +Generate or update pip requirement files +---------------------------------------- + +Scripts +~~~~~~~ + +**gen_requirements.py**: create/update requirements files from currently + installed requirements. + +**gen_requirements_dev.py** does the same but can subtract the main requirements + to get extra requirements used in only development. + + +Usage +~~~~~ + +The sequence of commands to run are: + + +* Start with these to generate the main pip requirements file:: + + ./configure --clean + ./configure + python etc/scripts/gen_requirements.py --site-packages-dir + +* You can optionally install or update extra main requirements after the + ./configure step such that these are included in the generated main requirements. + +* Optionally, generate a development pip requirements file by running these:: + + ./configure --clean + ./configure --dev + python etc/scripts/gen_requirements_dev.py --site-packages-dir + +* You can optionally install or update extra dev requirements after the + ./configure step such that these are included in the generated dev + requirements. + +Notes: we generate development requirements after the main as this step requires +the main requirements.txt to be up-to-date first. See **gen_requirements.py and +gen_requirements_dev.py** --help for details. + +Note: this does NOT hash requirements for now. + +Note: Be aware that if you are using "conditional" requirements (e.g. only for +OS or Python versions) in setup.py/setp.cfg/requirements.txt as these are NOT +yet supported. + + +Populate a thirdparty directory with wheels, sources, .ABOUT and license files +------------------------------------------------------------------------------ + +Scripts +~~~~~~~ + +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. + +* **check_thirdparty.py** will check a thirdparty directory for errors. + + +Upgrade virtualenv app +---------------------- + +The bundled virtualenv.pyz has to be upgraded by hand and is stored under +etc/thirdparty + +* Fetch https://github.com/pypa/get-virtualenv/raw//public/virtualenv.pyz + for instance https://github.com/pypa/get-virtualenv/raw/20.2.2/public/virtualenv.pyz + and save to thirdparty and update the ABOUT and LICENSE files as needed. + +* This virtualenv app contains also bundled pip, wheel and setuptools that are + essential for the installation to work. + + +Other files +=========== + +The other files and scripts are test, support and utility modules used by the +main scripts documented here. diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py new file mode 100644 index 00000000..65ae595e --- /dev/null +++ b/etc/scripts/check_thirdparty.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import click + +import utils_thirdparty + + +@click.command() +@click.option( + "-d", + "--dest", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + required=True, + help="Path to the thirdparty directory to check.", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) +@click.help_option("-h", "--help") +def check_thirdparty_dir( + dest, + wheels, + sdists, +): + """ + Check a thirdparty directory for problems and print these on screen. + """ + print("==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + check_thirdparty_dir() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100644 index 00000000..76a19a60 --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import sys +from collections import defaultdict + +import click + +import utils_requirements +import utils_thirdparty + +TRACE = False +TRACE_DEEP = False + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEX_URLS, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.option( + "--use-cached-index", + is_flag=True, + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", +) +@click.option( + "--sdist-only", + "sdist_only", + type=str, + metavar="SDIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in sdist format (no wheels). " + "The command will not fail and exit if no wheel exists for these names", +) +@click.option( + "--wheel-only", + "wheel_only", + type=str, + metavar="WHEEL", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that come only in wheel format (no sdist). " + "The command will not fail and exit if no sdist exists for these names", +) +@click.option( + "--no-dist", + "no_dist", + type=str, + metavar="DIST", + default=tuple(), + show_default=False, + multiple=True, + help="Package name(s) that do not come either in wheel or sdist format. " + "The command will not fail and exit if no distribution exists for these names", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, + use_cached_index, + sdist_only, + wheel_only, + no_dist, +): + """ + Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` + PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all + supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels + and sources fetched. + + Download from the provided PyPI simple --index-url INDEX(s) URLs. + """ + if not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_required_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if latest_version: + names = set(name for name, _version in sorted(required_name_versions)) + required_name_versions = {(n, None) for n in names} + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if TRACE_DEEP: + print("required_name_versions:") + for n, v in required_name_versions: + print(f" {n} @ {v}") + + # create the environments matrix we need for wheels + environments = None + if wheels: + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + # Collect PyPI repos + repos = [] + for index_url in index_urls: + index_url = index_url.strip("/") + existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) + if existing: + existing.use_cached_index = use_cached_index + repos.append(existing) + else: + repo = utils_thirdparty.PypiSimpleRepository( + index_url=index_url, + use_cached_index=use_cached_index, + ) + repos.append(repo) + + wheels_or_sdist_not_found = defaultdict(list) + + for name, version in sorted(required_name_versions): + nv = name, version + print(f"Processing: {name} @ {version}") + if wheels: + for environment in environments: + if TRACE: + print(f" ==> Fetching wheel for envt: {environment}") + + fetched = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + repos=repos, + ) + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) + if TRACE: + print(" NOT FOUND") + + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): + if TRACE: + print(f" ==> Fetching sdist: {name}=={version}") + + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + repos=repos, + ) + if not fetched: + wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") + if TRACE: + print(" NOT FOUND") + + mia = [] + for nv, dists in wheels_or_sdist_not_found.items(): + name, _, version = nv.partition("==") + if name in no_dist: + continue + sdist_missing = sdists and "sdist" in dists and name not in wheel_only + if sdist_missing: + mia.append(f"SDist missing: {nv} {dists}") + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only + if wheels_missing: + mia.append(f"Wheels missing: {nv} {dists}") + + if mia: + for m in mia: + print(m) + raise Exception(mia) + + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print("==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py new file mode 100644 index 00000000..89d06265 --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT +# Copyright (c) 2010 David Wolever . All rights reserved. +# originally from https://github.com/wolever/pip2pi + +import hashlib +import os +import re +import shutil +from collections import defaultdict +from html import escape +from pathlib import Path +from typing import NamedTuple + +""" +Generate a PyPI simple index froma directory. +""" + + +class InvalidDistributionFilename(Exception): + pass + + +def get_package_name_from_filename(filename): + """ + Return the normalized package name extracted from a package ``filename``. + Normalization is done according to distribution name rules. + Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: + + >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") + 'foo' + >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") + 'foo-bar' + >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") + 'cython' + >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") + 'python-ldap' + >>> try: + ... get_package_name_from_filename("foo.whl") + ... except InvalidDistributionFilename: + ... pass + >>> try: + ... get_package_name_from_filename("foo.png") + ... except InvalidDistributionFilename: + ... pass + """ + if not filename or not filename.endswith(dist_exts): + raise InvalidDistributionFilename(filename) + + filename = os.path.basename(filename) + + if filename.endswith(sdist_exts): + name_ver = None + extension = None + + for ext in sdist_exts: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + raise InvalidDistributionFilename(filename) + + name, _, version = name_ver.rpartition("-") + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(wheel_ext): + wheel_info = get_wheel_from_filename(filename) + + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group("name") + version = wheel_info.group("version") + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(app_ext): + name_ver, extension, _ = filename.rpartition(".pyz") + + if "-" in filename: + name, _, version = name_ver.rpartition("-") + else: + name = name_ver + + if not name: + raise InvalidDistributionFilename(filename) + + name = normalize_name(name) + return name + + +def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + +def build_per_package_index(pkg_name, packages, base_url): + """ + Return an HTML document as string representing the index for a package + """ + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in sorted(packages, key=lambda p: p.archive_file): + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = """ + + + Links for all packages + + """ + document.append(header) + + for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): + for package in sorted(packages, key=lambda p: p.archive_file): + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
    " + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): + """ + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. + + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links + """ + + directory = Path(directory) + + index_dir = directory / "simple" + if index_dir.exists(): + shutil.rmtree(str(index_dir), ignore_errors=True) + + index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) + + # generate the main simple index.html + simple_html_index = [ + "", + "PyPI Simple Index", + '', + ] + + for pkg_file in directory.iterdir(): + pkg_filename = pkg_file.name + + if ( + not pkg_file.is_file() + or not pkg_filename.endswith(dist_exts) + or pkg_filename.startswith(".") + ): + continue + + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) + pkg_index_dir = index_dir / pkg_name + pkg_index_dir.mkdir(parents=True, exist_ok=True) + pkg_indexed_file = pkg_index_dir / pkg_filename + + link_target = Path("../..") / pkg_filename + pkg_indexed_file.symlink_to(link_target) + + if pkg_name not in packages_by_package_name: + esc_name = escape(pkg_name) + simple_html_index.append(f'{esc_name}
    ') + + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + + # finalize main index + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) + +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) + +if __name__ == "__main__": + import sys + + pkg_dir = sys.argv[1] + build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_pypi_simple.py.ABOUT b/etc/scripts/gen_pypi_simple.py.ABOUT new file mode 100644 index 00000000..4de5ded0 --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: gen_pypi_simple.py +name: gen_pypi_simple.py +license_expression: bsd-2-clause-views and mit +copyright: Copyright (c) nexB Inc. + Copyright (c) 2010 David Wolever + Copyright (c) The pip developers +notes: Originally from https://github.com/wolever/pip2pi and modified extensivley + Also partially derived from pip code diff --git a/etc/scripts/gen_pypi_simple.py.NOTICE b/etc/scripts/gen_pypi_simple.py.NOTICE new file mode 100644 index 00000000..6e0fbbcd --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.NOTICE @@ -0,0 +1,56 @@ +SPDX-License-Identifier: BSD-2-Clause-Views AND mit + +Copyright (c) nexB Inc. +Copyright (c) 2010 David Wolever +Copyright (c) The pip developers + + +Original code: copyright 2010 David Wolever . All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of David Wolever. + + +Original code: Copyright (c) 2008-2020 The pip developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py new file mode 100644 index 00000000..1b879442 --- /dev/null +++ b/etc/scripts/gen_requirements.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import argparse +import pathlib + +import utils_requirements + +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" + + +def gen_requirements(): + description = """ + Create or replace the `--requirements-file` file FILE requirements file with all + locally installed Python packages.all Python packages found installed in `--site-packages-dir` + """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + + utils_requirements.lock_requirements( + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, + ) + + +if __name__ == "__main__": + gen_requirements() diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py new file mode 100644 index 00000000..85482056 --- /dev/null +++ b/etc/scripts/gen_requirements_dev.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import argparse +import pathlib + +import utils_requirements + +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" + + +def gen_dev_requirements(): + description = """ + Create or overwrite the `--dev-requirements-file` pip requirements FILE with + all Python packages found installed in `--site-packages-dir`. Exclude + package names also listed in the --main-requirements-file pip requirements + FILE (that are assume to the production requirements and therefore to always + be present in addition to the development requirements). + """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + + utils_requirements.lock_dev_requirements( + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, + ) + + +if __name__ == "__main__": + gen_dev_requirements() diff --git a/etc/scripts/gpl-2.0.LICENSE b/etc/scripts/gpl-2.0.LICENSE deleted file mode 100644 index d815b29b..00000000 --- a/etc/scripts/gpl-2.0.LICENSE +++ /dev/null @@ -1,339 +0,0 @@ -GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. \ No newline at end of file diff --git a/etc/scripts/irc-notify.py b/etc/scripts/irc-notify.py deleted file mode 100644 index 8b8376f2..00000000 --- a/etc/scripts/irc-notify.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -From: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py -and: https://github.com/gridsync/gridsync - -Copyright (C) 2015-2016 Christopher R. Wood - -This program is free software; you can redistribute it and/or modify it under the -terms of the GNU General Public License as published by the Free Software Foundation; -either version 2 of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with this -program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, -Fifth Floor, Boston, MA 02110-1301 USA. - -Simple AppVeyor IRC notification script. - -Modified by nexB on October 2016: - - rework the handling of environment variables. - - made the script use functions - - support only Appveyor loading its environment variable to craft IRC notices. - - -The first argument is a Frenode channel. Other arguments passed to the script will be -sent as notice messages content and any {var}-formatted environment variables will be -expanded automatically, replaced with a corresponding Appveyor environment variable -value. se commas to delineate multiple messages. - - -Example: -export APPVEYOR_URL=https://ci.appveyor.com -export APPVEYOR_PROJECT_NAME=attributecode -export APPVEYOR_REPO_COMMIT_AUTHOR=pombredanne -export APPVEYOR_REPO_COMMIT_TIMESTAMP=2016-10-31 -export APPVEYOR_REPO_PROVIDER=gihub -export APPVEYOR_REPO_BRANCH=repo_branch -export APPVEYOR_PULL_REQUEST_TITLE=pull_request_title -export APPVEYOR_BUILD_VERSION=1 -export APPVEYOR_REPO_COMMIT=22c95b72e29248dc4de9b85e590ee18f6f587de8 -export APPVEYOR_REPO_COMMIT_MESSAGE="some IRC test" -export APPVEYOR_ACCOUNT_NAME=nexB -export APPVEYOR_PULL_REQUEST_NUMBER=pull_request_number -export APPVEYOR_REPO_NAME=nexB/attributecode -python etc/scripts/irc-notify.py aboutcode '[{project_name}:{branch}] {short_commit}: "{message}" ({author}) {color_red}Succeeded','Details: {build_url} | Commit: {commit_url}' - -See also https://github.com/gridsync/gridsync/blob/master/appveyor.yml for examples -in Appveyor's YAML: - - on_success: - - "python etc/scripts/irc-notify.py channel [{project_name}:{branch}] {short_commit}: \"{message}\" ({author}) {color_green}Succeeded,Details: {build_url},Commit: {commit_url}" - on_failure: - - "python etc/scripts/irc-notify.py channel [{project_name}:{branch}] {short_commit}: \"{message}\" ({author}) {color_red}Failed,Details: {build_url},Commit: {commit_url}" - -""" - -import os, random, socket, ssl, sys, time - - -def appveyor_vars(): - """ - Return a dict of key value carfted from appveyor environment variables. - """ - from os import environ - - appveyor_url = environ.get('APPVEYOR_URL') - message_extended = environ.get('APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED') - branch = environ.get('APPVEYOR_REPO_BRANCH') - author = environ.get('APPVEYOR_REPO_COMMIT_AUTHOR') - author_email = environ.get('APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL') - timestamp = environ.get('APPVEYOR_REPO_COMMIT_TIMESTAMP') - repo_provider = environ.get('APPVEYOR_REPO_PROVIDER') - project_name = environ.get('APPVEYOR_PROJECT_NAME') - pull_request_title = environ.get('APPVEYOR_PULL_REQUEST_TITLE') - build_version = environ.get('APPVEYOR_BUILD_VERSION') - commit = environ.get('APPVEYOR_REPO_COMMIT') - message = environ.get('APPVEYOR_REPO_COMMIT_MESSAGE') - account_name = environ.get('APPVEYOR_ACCOUNT_NAME') - pull_request_number = environ.get('APPVEYOR_PULL_REQUEST_NUMBER') - repo_name = environ.get('APPVEYOR_REPO_NAME') - - short_commit = commit[:7] - build_url = '{appveyor_url}/project/{account_name}/{project_name}/build/{build_version}'.format(**locals()) - commit_url = 'https://{repo_provider}.com/{repo_name}/commit/{commit}'.format(**locals()) - - vars = dict( - appveyor_url=appveyor_url, - account_name=account_name, - project_name=project_name, - build_version=build_version, - - build_url=build_url, - - repo_provider=repo_provider, - repo_name=repo_name, - branch=branch, - author=author, - author_email=author_email, - timestamp=timestamp, - commit=commit, - short_commit=short_commit, - message=message, - message_extended=message_extended, - - pull_request_title=pull_request_title, - pull_request_number=pull_request_number, - - commit_url=commit_url, - - color_green='\x033', - color_red='\x034', - ) - return vars - - -if __name__ == '__main__': - apvy_vars = appveyor_vars() - - channel = sys.argv[1] - messages = sys.argv[2:] - messages = ' '.join(messages) - messages = messages.split(',') - messages = [msg.format(**apvy_vars).strip() for msg in messages] - - irc_username = 'Appveyor' - irc_nick = irc_username + str(random.randint(1, 9999)) - - # establish connection - irc_sock = ssl.wrap_socket(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) - irc_sock.connect((socket.gethostbyname('chat.freenode.net'), 6697)) - irc_sock.send('NICK {0}\r\nUSER {0} * 0 :{0}\r\n'.format(irc_username).encode()) - irc_file = irc_sock.makefile() - - while irc_file: - line = irc_file.readline() - print(line.rstrip()) - response = line.split() - - if response[0] == 'PING': - irc_file.send('PONG {}\r\n'.format(reponse[1]).encode()) - - elif response[1] == '433': - irc_sock.send('NICK {}\r\n'.format(irc_nick).encode()) - - elif response[1] == '001': - time.sleep(5) - # send notification - for msg in messages: - print('NOTICE #{} :{}'.format(channel, msg)) - irc_sock.send('NOTICE #{} :{}\r\n'.format(channel, msg).encode()) - time.sleep(5) - sys.exit() diff --git a/etc/scripts/irc-notify.py.ABOUT b/etc/scripts/irc-notify.py.ABOUT deleted file mode 100644 index 9d56add7..00000000 --- a/etc/scripts/irc-notify.py.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: irc-notify.py -name: irc-notify.py -version: def54f8166089b733d166665fdabcad4cdc526d8 -copyright: Copyright (C) 2015-2016 Christopher R. Wood -description: Quick and dirty IRC notification script. -download_url: https://raw.githubusercontent.com/gridsync/gridsync/def54f8166089b733d166665fdabcad4cdc526d8/misc/irc-notify.py -homepage_url: https://github.com/gridsync/gridsync -license_expression: gpl-2.0 -licenses: - - file: gpl-2.0.LICENSE - key: gpl-2.0 - name: GNU General Public License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:gpl-2.0 -notice_file: irc-notify.py.NOTICE -owner: Christopher R. Wood diff --git a/etc/scripts/irc-notify.py.NOTICE b/etc/scripts/irc-notify.py.NOTICE deleted file mode 100644 index d73fc871..00000000 --- a/etc/scripts/irc-notify.py.NOTICE +++ /dev/null @@ -1,12 +0,0 @@ -This program is free software; you can redistribute it and/or modify it under the -terms of the GNU General Public License as published by the Free Software -Foundation; either version 2 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT ANY -WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A -PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with this -program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, -Fifth Floor, Boston, MA 02110-1301 USA. \ No newline at end of file diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt new file mode 100644 index 00000000..7c514da9 --- /dev/null +++ b/etc/scripts/requirements.txt @@ -0,0 +1,12 @@ +aboutcode_toolkit +attrs +commoncode +click +requests +saneyaml +pip +setuptools +twine +wheel +build +packvers diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py new file mode 100644 index 00000000..0e9c360a --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -0,0 +1,131 @@ +""" +Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import sysconfig +from unittest.mock import patch + +import pytest + +import utils_pip_compatibility_tags + + +@pytest.mark.parametrize( + "version_info, expected", + [ + ((2,), "2"), + ((2, 8), "28"), + ((3,), "3"), + ((3, 6), "36"), + # Test a tuple of length 3. + ((3, 6, 5), "36"), + # Test a 2-digit minor version. + ((3, 10), "310"), + ], +) +def test_version_info_to_nodot(version_info, expected): + actual = utils_pip_compatibility_tags.version_info_to_nodot(version_info) + assert actual == expected + + +class Testcompatibility_tags: + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + get_config_var = sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + + return _mock_get_config_var + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + import pip._internal.utils.compatibility_tags + + mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") + + with patch("sysconfig.get_config_var", mock_gcf): + supported = pip._internal.utils.compatibility_tags.get_supported() + + for tag in supported: + assert "-" not in tag.interpreter + assert "-" not in tag.abi + assert "-" not in tag.platform + + +class TestManylinux2010Tags: + @pytest.mark.parametrize( + "manylinux2010,manylinux1", + [ + ("manylinux2010_x86_64", "manylinux1_x86_64"), + ("manylinux2010_i686", "manylinux1_i686"), + ], + ) + def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): + """ + Specifying manylinux2010 implies manylinux1. + """ + groups = {} + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) + for tag in supported: + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) + + for arches in groups.values(): + if arches == ["any"]: + continue + assert arches[:2] == [manylinux2010, manylinux1] + + +class TestManylinux2014Tags: + @pytest.mark.parametrize( + "manylinuxA,manylinuxB", + [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ], + ) + def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): + """ + Specifying manylinux2014 implies manylinux2010/manylinux1. + """ + groups = {} + supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinuxA]) + for tag in supported: + groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) + + expected_arches = [manylinuxA] + expected_arches.extend(manylinuxB) + for arches in groups.values(): + if arches == ["any"]: + continue + assert arches[:3] == expected_arches diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 00000000..07eee35a --- /dev/null +++ b/etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: test_utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: tests/unit/test_utils_compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#tests/unit/test_utils_compatibility_tags.py + +download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling diff --git a/etc/scripts/test_utils_pypi_supported_tags.py b/etc/scripts/test_utils_pypi_supported_tags.py new file mode 100644 index 00000000..d291572d --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from utils_pypi_supported_tags import validate_platforms_for_pypi + +""" +Wheel platform checking tests + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +""" + + +def validate_wheel_filename_for_pypi(filename): + """ + Validate if the filename is a PyPI/warehouse-uploadable wheel file name + with supported platform tags. Return a list of unsupported platform tags or + an empty list if all tags are supported. + """ + from utils_thirdparty import Wheel + + wheel = Wheel.from_filename(filename) + return validate_platforms_for_pypi(wheel.platforms) + + +@pytest.mark.parametrize( + "plat", + [ + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_i686", + "manylinux1_x86_64", + "manylinux2010_i686", + "manylinux2010_x86_64", + "manylinux2014_i686", + "manylinux2014_x86_64", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "manylinux_2_5_i686", + "manylinux_2_12_x86_64", + "manylinux_2_17_aarch64", + "manylinux_2_17_armv7l", + "manylinux_2_17_ppc64", + "manylinux_2_17_ppc64le", + "manylinux_3_0_s390x", + "macosx_10_6_intel", + "macosx_10_13_x86_64", + "macosx_11_0_x86_64", + "macosx_10_15_arm64", + "macosx_11_10_universal2", + # A real tag used by e.g. some numpy wheels + ( + "macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." + "macosx_10_10_intel.macosx_10_10_x86_64" + ), + ], +) +def test_is_valid_pypi_wheel_return_true_for_supported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + assert not validate_wheel_filename_for_pypi(filename) + + +@pytest.mark.parametrize( + "plat", + [ + "linux_x86_64", + "linux_x86_64.win32", + "macosx_9_2_x86_64", + "macosx_12_2_arm64", + "macosx_10_15_amd64", + ], +) +def test_is_valid_pypi_wheel_raise_exception_for_aunsupported_wheel(plat): + filename = f"foo-1.2.3-cp34-none-{plat}.whl" + invalid = validate_wheel_filename_for_pypi(filename) + assert invalid diff --git a/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 00000000..176efacc --- /dev/null +++ b/etc/scripts/test_utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: test_utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: tests/unit/forklift/test_legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#tests/unit/forklift/test_legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Test for wheel platform checking copied and heavily modified on + 2020-12-24 from warehouse. This contains the basic functions to check if a + wheel file name is would be supported for uploading to PyPI. diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py new file mode 100644 index 00000000..374c06f2 --- /dev/null +++ b/etc/scripts/update_skeleton.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. AboutCode, and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import os +import subprocess + +import click + + +ABOUTCODE_PUBLIC_REPO_NAMES = [ + "aboutcode-toolkit", + "ahocode", + "bitcode", + "clearcode-toolkit", + "commoncode", + "container-inspector", + "debian-inspector", + "deltacode", + "elf-inspector", + "extractcode", + "fetchcode", + "gemfileparser2", + "gh-issue-sandbox", + "go-inspector", + "heritedcode", + "license-expression", + "license_copyright_pipeline", + "nuget-inspector", + "pip-requirements-parser", + "plugincode", + "purldb", + "pygmars", + "python-inspector", + "sanexml", + "saneyaml", + "scancode-analyzer", + "scancode-toolkit-contrib", + "scancode-toolkit-reference-scans", + "thirdparty-toolkit", + "tracecode-toolkit", + "tracecode-toolkit-strace", + "turbo-spdx", + "typecode", + "univers", +] + + +@click.command() +@click.help_option("-h", "--help") +def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): + """ + Update project files of AboutCode projects that use the skeleton + + This script will: + - Clone the repo + - Add the skeleton repo as a new origin + - Create a new branch named "update-skeleton-files" + - Merge in the new skeleton files into the "update-skeleton-files" branch + + The user will need to save merge commit messages that pop up when running + this script in addition to resolving the merge conflicts on repos that have + them. + """ + + # Create working directory + work_dir_path = Path("/tmp/update_skeleton/") + if not os.path.exists(work_dir_path): + os.makedirs(work_dir_path, exist_ok=True) + + for repo_name in repo_names: + # Move to work directory + os.chdir(work_dir_path) + + # Clone repo + repo_git = f"git@github.com:aboutcode-org/{repo_name}.git" + subprocess.run(["git", "clone", repo_git]) + + # Go into cloned repo + os.chdir(work_dir_path / repo_name) + + # Add skeleton as an origin + subprocess.run( + ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] + ) + + # Fetch skeleton files + subprocess.run(["git", "fetch", "skeleton"]) + + # Create and checkout new branch + subprocess.run(["git", "checkout", "-b", "update-skeleton-files"]) + + # Merge skeleton files into the repo + subprocess.run(["git", "merge", "skeleton/main", "--allow-unrelated-histories"]) + + +if __name__ == "__main__": + update_skeleton_files() diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py new file mode 100644 index 00000000..b6bff518 --- /dev/null +++ b/etc/scripts/utils_dejacode.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import io +import os +import zipfile + +import requests +import saneyaml +from packvers import version as packaging_version + +""" +Utility to create and retrieve package and ABOUT file data from DejaCode. +""" + +DEJACODE_API_KEY = os.environ.get("DEJACODE_API_KEY", "") +DEJACODE_API_URL = os.environ.get("DEJACODE_API_URL", "") + +DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" +DEJACODE_API_HEADERS = { + "Authorization": f"Token {DEJACODE_API_KEY}", + "Accept": "application/json; indent=4", +} + + +def can_do_api_calls(): + if not DEJACODE_API_KEY and DEJACODE_API_URL: + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + return False + else: + return True + + +def fetch_dejacode_packages(params): + """ + Return a list of package data mappings calling the package API with using + `params` or an empty list. + """ + if not can_do_api_calls(): + return [] + + response = requests.get( + DEJACODE_API_URL_PACKAGES, + params=params, + headers=DEJACODE_API_HEADERS, + timeout=10, + ) + + return response.json()["results"] + + +def get_package_data(distribution): + """ + Return a mapping of package data or None for a Distribution `distribution`. + """ + results = fetch_dejacode_packages(distribution.identifiers()) + + len_results = len(results) + + if len_results == 1: + return results[0] + + elif len_results > 1: + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + else: + print("Could not find package:", distribution.download_url) + + +def update_with_dejacode_data(distribution): + """ + Update the Distribution `distribution` with DejaCode package data. Return + True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + return distribution.update(package_data, keep_extra=False) + + print(f"No package found for: {distribution}") + + +def update_with_dejacode_about_data(distribution): + """ + Update the Distribution `distribution` wiht ABOUT code data fetched from + DejaCode. Return True if data was updated. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about" + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) + # note that this is YAML-formatted + about_text = response.json()["about_data"] + about_data = saneyaml.load(about_text) + + return distribution.update(about_data, keep_extra=True) + + print(f"No package found for: {distribution}") + + +def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): + """ + Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched + from DejaCode for a Distribution `distribution`. Return True if files were + fetched. + """ + package_data = get_package_data(distribution) + if package_data: + package_api_url = package_data["api_url"] + about_url = f"{package_api_url}about_files" + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) + about_zip = response.content + with io.BytesIO(about_zip) as zf: + with zipfile.ZipFile(zf) as zi: + zi.extractall(path=dest_dir) + return True + + print(f"No package found for: {distribution}") + + +def find_latest_dejacode_package(distribution): + """ + Return a mapping of package data for the closest version to + a Distribution `distribution` or None. + Return the newest of the packages if prefer_newest is True. + Filter out version-specific attributes. + """ + ids = distribution.purl_identifiers(skinny=True) + packages = fetch_dejacode_packages(params=ids) + if not packages: + return + + for package_data in packages: + matched = ( + package_data["download_url"] == distribution.download_url + and package_data["version"] == distribution.version + and package_data["filename"] == distribution.filename + ) + + if matched: + return package_data + + # there was no exact match, find the latest version + # TODO: consider the closest version rather than the latest + # or the version that has the best data + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] + with_versions = sorted(with_versions) + latest_version, latest_package_version = sorted(with_versions)[-1] + print( + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", + ) + + return latest_package_version + + +def create_dejacode_package(distribution): + """ + Create a new DejaCode Package a Distribution `distribution`. + Return the new or existing package data. + """ + if not can_do_api_calls(): + return + + existing_package_data = get_package_data(distribution) + if existing_package_data: + return existing_package_data + + print(f"Creating new DejaCode package for: {distribution}") + + new_package_payload = { + # Trigger data collection, scan, and purl + "collect_data": 1, + } + + fields_to_carry_over = [ + "download_urltype", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + ] + + for field in fields_to_carry_over: + value = getattr(distribution, field, None) + if value: + new_package_payload[field] = value + + response = requests.post( + DEJACODE_API_URL_PACKAGES, + data=new_package_payload, + headers=DEJACODE_API_HEADERS, + timeout=10, + ) + new_package_data = response.json() + if response.status_code != 201: + raise Exception(f"Error, cannot create package for: {distribution}") + + print(f"New Package created at: {new_package_data['absolute_url']}") + return new_package_data diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py new file mode 100644 index 00000000..dd954bca --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -0,0 +1,192 @@ +""" +Generate and work with PEP 425 Compatibility Tags. + +copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import re + +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms + +_osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") + + +def version_info_to_nodot(version_info): + # type: (Tuple[int, ...]) -> str + # Only use up to the first two numbers. + return "".join(map(str, version_info[:2])) + + +def _mac_platforms(arch): + # type: (str) -> List[str] + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + mac_version = (int(major), int(minor)) + arches = [ + # Since we have always only checked that the platform starts + # with "macosx", for backwards-compatibility we extract the + # actual prefix provided by the user in case they provided + # something like "macosxcustom_". It may be good to remove + # this as undocumented or deprecate it in the future. + "{}_{}".format(name, arch[len("macosx_") :]) + for arch in mac_platforms(mac_version, actual_arch) + ] + else: + # arch pattern didn't match (?!) + arches = [arch] + return arches + + +def _custom_manylinux_platforms(arch): + # type: (str) -> List[str] + arches = [arch] + arch_prefix, arch_sep, arch_suffix = arch.partition("_") + if arch_prefix == "manylinux2014": + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {"i686", "x86_64"}: + arches.append("manylinux2010" + arch_sep + arch_suffix) + arches.append("manylinux1" + arch_sep + arch_suffix) + elif arch_prefix == "manylinux2010": + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches.append("manylinux1" + arch_sep + arch_suffix) + return arches + + +def _get_custom_platforms(arch): + # type: (str) -> List[str] + arch_prefix, _arch_sep, _arch_suffix = arch.partition("_") + if arch.startswith("macosx"): + arches = _mac_platforms(arch) + elif arch_prefix in ["manylinux2014", "manylinux2010"]: + arches = _custom_manylinux_platforms(arch) + else: + arches = [arch] + return arches + + +def _expand_allowed_platforms(platforms): + # type: (Optional[List[str]]) -> Optional[List[str]] + if not platforms: + return None + + seen = set() + result = [] + + for p in platforms: + if p in seen: + continue + additions = [c for c in _get_custom_platforms(p) if c not in seen] + seen.update(additions) + result.extend(additions) + + return result + + +def _get_python_version(version): + # type: (str) -> PythonVersion + if len(version) > 1: + return int(version[0]), int(version[1:]) + else: + return (int(version[0]),) + + +def _get_custom_interpreter(implementation=None, version=None): + # type: (Optional[str], Optional[str]) -> str + if implementation is None: + implementation = interpreter_name() + if version is None: + version = interpreter_version() + return f"{implementation}{version}" + + +def get_supported( + version=None, # type: Optional[str] + platforms=None, # type: Optional[List[str]] + impl=None, # type: Optional[str] + abis=None, # type: Optional[List[str]] +): + # type: (...) -> List[Tag] + """ + Return a list of supported tags for each version specified in + `versions`. + + :param version: a string version, of the form "33" or "32", + or None. The version will be assumed to support our ABI. + :param platforms: specify a list of platforms you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abis: specify a list of abis you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] # type: List[Tag] + + python_version = None # type: Optional[PythonVersion] + if version is not None: + python_version = _get_python_version(version) + + interpreter = _get_custom_interpreter(impl, version) + + platforms = _expand_allowed_platforms(platforms) + + is_cpython = (impl or interpreter_name()) == "cp" + if is_cpython: + supported.extend( + cpython_tags( + python_version=python_version, + abis=abis, + platforms=platforms, + ) + ) + else: + supported.extend( + generic_tags( + interpreter=interpreter, + abis=abis, + platforms=platforms, + ) + ) + supported.extend( + compatible_tags( + python_version=python_version, + interpreter=interpreter, + platforms=platforms, + ) + ) + + return supported diff --git a/etc/scripts/utils_pip_compatibility_tags.py.ABOUT b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT new file mode 100644 index 00000000..7bbb026b --- /dev/null +++ b/etc/scripts/utils_pip_compatibility_tags.py.ABOUT @@ -0,0 +1,14 @@ +about_resource: utils_pip_compatibility_tags.py + +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/utils/compatibility_tags.py + +package_url: pkg:github/pypa/pip@20.3.1#src/pip/_internal/utils/compatibility_tags.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: subset copied from pip for tag handling \ No newline at end of file diff --git a/etc/scripts/utils_pypi_supported_tags.py b/etc/scripts/utils_pypi_supported_tags.py new file mode 100644 index 00000000..de9f21b6 --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py @@ -0,0 +1,105 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +""" +Wheel platform checking + +Copied and modified on 2020-12-24 from +https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py + +This contains the basic functions to check if a wheel file name is would be +supported for uploading to PyPI. +""" + +# These platforms can be handled by a simple static list: +_allowed_platforms = { + "any", + "win32", + "win_amd64", + "win_ia64", + "manylinux1_x86_64", + "manylinux1_i686", + "manylinux2010_x86_64", + "manylinux2010_i686", + "manylinux2014_x86_64", + "manylinux2014_i686", + "manylinux2014_aarch64", + "manylinux2014_armv7l", + "manylinux2014_ppc64", + "manylinux2014_ppc64le", + "manylinux2014_s390x", + "linux_armv6l", + "linux_armv7l", +} +# macosx is a little more complicated: +_macosx_platform_re = re.compile(r"macosx_(?P\d+)_(\d+)_(?P.*)") +_macosx_arches = { + "ppc", + "ppc64", + "i386", + "x86_64", + "arm64", + "intel", + "fat", + "fat32", + "fat64", + "universal", + "universal2", +} +_macosx_major_versions = { + "10", + "11", +} + +# manylinux pep600 is a little more complicated: +_manylinux_platform_re = re.compile(r"manylinux_(\d+)_(\d+)_(?P.*)") +_manylinux_arches = { + "x86_64", + "i686", + "aarch64", + "armv7l", + "ppc64", + "ppc64le", + "s390x", +} + + +def is_supported_platform_tag(platform_tag): + """ + Return True if the ``platform_tag`` is supported on PyPI. + """ + if platform_tag in _allowed_platforms: + return True + m = _macosx_platform_re.match(platform_tag) + if m and m.group("major") in _macosx_major_versions and m.group("arch") in _macosx_arches: + return True + m = _manylinux_platform_re.match(platform_tag) + if m and m.group("arch") in _manylinux_arches: + return True + return False + + +def validate_platforms_for_pypi(platforms): + """ + Validate if the wheel platforms are supported platform tags on Pypi. Return + a list of unsupported platform tags or an empty list if all tags are + supported. + """ + + # Check that if it's a binary wheel, it's on a supported platform + invalid_tags = [] + for plat in platforms: + if not is_supported_platform_tag(plat): + invalid_tags.append(plat) + return invalid_tags diff --git a/etc/scripts/utils_pypi_supported_tags.py.ABOUT b/etc/scripts/utils_pypi_supported_tags.py.ABOUT new file mode 100644 index 00000000..228a538a --- /dev/null +++ b/etc/scripts/utils_pypi_supported_tags.py.ABOUT @@ -0,0 +1,17 @@ +about_resource: utils_pypi_supported_tags.py + +type: github +namespace: pypa +name: warehouse +version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d +subpath: warehouse/forklift/legacy.py + +package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#warehouse/forklift/legacy.py + +download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py +copyright: Copyright (c) The warehouse developers +homepage_url: https://warehouse.readthedocs.io +license_expression: apache-2.0 +notes: Wheel platform checking copied and heavily modified on 2020-12-24 from + warehouse. This contains the basic functions to check if a wheel file name is + would be supported for uploading to PyPI. diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py new file mode 100644 index 00000000..f377578e --- /dev/null +++ b/etc/scripts/utils_requirements.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +import re +import subprocess + +""" +Utilities to manage requirements files and call pip. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for bootstrapping with no requirements installed. +""" + + +def load_requirements(requirements_file="requirements.txt", with_unpinned=False): + """ + Yield package (name, version) tuples for each requirement in a `requirement` + file. Only accept requirements pinned to an exact version. + """ + with open(requirements_file) as reqs: + req_lines = reqs.read().splitlines(False) + return get_required_name_versions(req_lines, with_unpinned=with_unpinned) + + +def get_required_name_versions(requirement_lines, with_unpinned=False): + """ + Yield required (name, version) tuples given a `requirement_lines` iterable of + requirement text lines. Only accept requirements pinned to an exact version. + """ + + for req_line in requirement_lines: + req_line = req_line.strip() + if not req_line or req_line.startswith("#"): + continue + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): + print(f"Requirement line is not supported: ignored: {req_line}") + continue + yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) + + +def get_required_name_version(requirement, with_unpinned=False): + """ + Return a (name, version) tuple given a `requirement` specifier string. + Requirement version must be pinned. If ``with_unpinned`` is True, unpinned + requirements are accepted and only the name portion is returned. + + For example: + >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") + >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") + >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") + >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected + >>> try: + ... assert not get_required_name_version("foo", with_unpinned=False) + ... except Exception as e: + ... assert "Requirement version must be pinned" in str(e) + """ + requirement = requirement and "".join(requirement.lower().split()) + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") + name, operator, version = split_req(requirement) + if not name: + raise ValueError(f"Name is required: {requirement}") + is_pinned = operator == "==" + if with_unpinned: + version = "" + else: + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") + return name, version + + +def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): + """ + Freeze and lock current installed requirements and save this to the + `requirements_file` requirements file. + """ + with open(requirements_file, "w") as fo: + fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) + + +def lock_dev_requirements( + dev_requirements_file="requirements-dev.txt", + main_requirements_file="requirements.txt", + site_packages_dir=None, +): + """ + Freeze and lock current installed development-only requirements and save + this to the `dev_requirements_file` requirements file. Development-only is + achieved by subtracting requirements from the `main_requirements_file` + requirements file from the current requirements using package names (and + ignoring versions). + """ + main_names = {n for n, _v in load_requirements(main_requirements_file)} + all_reqs = get_installed_reqs(site_packages_dir=site_packages_dir) + all_req_lines = all_reqs.splitlines(False) + all_req_nvs = get_required_name_versions(all_req_lines) + dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} + + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + with open(dev_requirements_file, "w") as fo: + fo.write(new_reqs) + + +def get_installed_reqs(site_packages_dir): + """ + Return the installed pip requirements as text found in `site_packages_dir` + as a text. + """ + if not os.path.exists(site_packages_dir): + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") + # Also include these packages in the output with --all: wheel, distribute, + # setuptools, pip + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 + + +comparators = ( + "===", + "~=", + "!=", + "==", + "<=", + ">=", + ">", + "<", +) + +_comparators_re = r"|".join(comparators) +version_splitter = re.compile(rf"({_comparators_re})") + + +def split_req(req): + """ + Return a three-tuple of (name, comparator, version) given a ``req`` + requirement specifier string. Each segment may be empty. Spaces are removed. + + For example: + >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") + >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") + >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") + >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") + >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") + """ + if not req: + raise ValueError("req is required") + # do not allow multiple constraints and tags + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") + req = "".join(req.split()) + if not any(c in req for c in comparators): + return req, "", "" + segments = version_splitter.split(req, maxsplit=1) + return tuple(segments) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py new file mode 100644 index 00000000..494fcb3b --- /dev/null +++ b/etc/scripts/utils_thirdparty.py @@ -0,0 +1,2288 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import email +import itertools +import os +import re +import shutil +import subprocess +import tempfile +import time +import urllib +from collections import defaultdict +from urllib.parse import quote_plus + +import attr +import license_expression +import packageurl +import requests +import saneyaml +from commoncode import fileutils +from commoncode.hash import multi_checksums +from commoncode.text import python_safe_name +from packvers import tags as packaging_tags +from packvers import version as packaging_version + +import utils_pip_compatibility_tags + +""" +Utilities to manage Python thirparty libraries source, binaries and metadata in +local directories and remote repositories. + +- download wheels for packages for all each supported operating systems + (Linux, macOS, Windows) and Python versions (3.x) combinations + +- download sources for packages (aka. sdist) + +- create, update and download ABOUT, NOTICE and LICENSE metadata for these + wheels and source distributions + +- update pip requirement files based on actually installed packages for + production and development + + +Approach +-------- + +The processing is organized around these key objects: + +- A PyPiPackage represents a PyPI package with its name and version and the + metadata used to populate an .ABOUT file and document origin and license. + It contains the downloadable Distribution objects for that version: + + - one Sdist source Distribution + - a list of Wheel binary Distribution + +- A Distribution (either a Wheel or Sdist) is identified by and created from its + filename as well as its name and version. + A Distribution is fetched from a Repository. + Distribution metadata can be loaded from and dumped to ABOUT files. + +- A Wheel binary Distribution can have Python/Platform/OS tags it supports and + was built for and these tags can be matched to an Environment. + +- An Environment is a combination of a Python version and operating system + (e.g., platfiorm and ABI tags.) and is represented by the "tags" it supports. + +- A plain LinksRepository which is just a collection of URLs scrape from a web + page such as HTTP diretory listing. It is used either with pip "--find-links" + option or to fetch ABOUT and LICENSE files. + +- A PypiSimpleRepository is a PyPI "simple" index where a HTML page is listing + package name links. Each such link points to an HTML page listing URLs to all + wheels and sdsist of all versions of this package. + +PypiSimpleRepository and Packages are related through packages name, version and +filenames. + +The Wheel models code is partially derived from the mit-licensed pip and the +Distribution/Wheel/Sdist design has been heavily inspired by the packaging- +dists library https://github.com/uranusjr/packaging-dists by Tzu-ping Chung +""" + +""" +Wheel downloader + +- parse requirement file +- create a TODO queue of requirements to process +- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} + + +- while we have package reqs in TODO queue, process one requirement: + - for each PyPI simple index: + - fetch through cache the PyPI simple index for this package + - for each environment: + - find a wheel matching pinned requirement in this index + - if file exist locally, continue + - fetch the wheel for env + - IF pure, break, no more needed for env + - collect requirement deps from wheel metadata and add to queue + - if fetched, break, otherwise display error message + + +""" + +TRACE = False +TRACE_DEEP = False +TRACE_ULTRA_DEEP = False + +# Supported environments +PYTHON_VERSIONS = "39", "310", "311", "312", "313", "314" + +PYTHON_DOT_VERSIONS_BY_VER = { + "39": "3.9", + "310": "3.10", + "311": "3.11", + "312": "3.12", + "313": "3.13", + "314": "3.14", +} + + +def get_python_dot_version(version): + """ + Return a dot version from a plain, non-dot version. + """ + return PYTHON_DOT_VERSIONS_BY_VER[version] + + +ABIS_BY_PYTHON_VERSION = { + "39": ["cp39", "cp39m", "abi3"], + "310": ["cp310", "cp310m", "abi3"], + "311": ["cp311", "cp311m", "abi3"], + "312": ["cp312", "cp312m", "abi3"], + "313": ["cp313", "cp313m", "abi3"], + "314": ["cp314", "cp314m", "abi3"], +} + +PLATFORMS_BY_OS = { + "linux": [ + "linux_x86_64", + "manylinux1_x86_64", + "manylinux2010_x86_64", + "manylinux2014_x86_64", + ], + "macos": [ + "macosx_10_6_intel", + "macosx_10_6_x86_64", + "macosx_10_9_intel", + "macosx_10_9_x86_64", + "macosx_10_10_intel", + "macosx_10_10_x86_64", + "macosx_10_11_intel", + "macosx_10_11_x86_64", + "macosx_10_12_intel", + "macosx_10_12_x86_64", + "macosx_10_13_intel", + "macosx_10_13_x86_64", + "macosx_10_14_intel", + "macosx_10_14_x86_64", + "macosx_10_15_intel", + "macosx_10_15_x86_64", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_11_0_x86_64", + "macosx_11_intel", + "macosx_10_9_universal2", + "macosx_10_10_universal2", + "macosx_10_11_universal2", + "macosx_10_12_universal2", + "macosx_10_13_universal2", + "macosx_10_14_universal2", + "macosx_10_15_universal2", + "macosx_11_0_universal2", + # 'macosx_11_0_arm64', + ], + "windows": [ + "win_amd64", + ], +} + +THIRDPARTY_DIR = "thirdparty" +CACHE_THIRDPARTY_DIR = ".cache/thirdparty" + +################################################################################ + +ABOUT_BASE_URL = "https://thirdparty.aboutcode.org/pypi" +ABOUT_PYPI_SIMPLE_URL = f"{ABOUT_BASE_URL}/simple" +ABOUT_LINKS_URL = f"{ABOUT_PYPI_SIMPLE_URL}/links.html" +PYPI_SIMPLE_URL = "https://pypi.org/simple" +PYPI_INDEX_URLS = (PYPI_SIMPLE_URL, ABOUT_PYPI_SIMPLE_URL) + +################################################################################ + +EXTENSIONS_APP = (".pyz",) +EXTENSIONS_SDIST = ( + ".tar.gz", + ".zip", + ".tar.xz", +) +EXTENSIONS_INSTALLABLE = EXTENSIONS_SDIST + (".whl",) +EXTENSIONS_ABOUT = ( + ".ABOUT", + ".LICENSE", + ".NOTICE", +) +EXTENSIONS = EXTENSIONS_INSTALLABLE + EXTENSIONS_ABOUT + EXTENSIONS_APP + +LICENSEDB_API_URL = "https://scancode-licensedb.aboutcode.org" + +LICENSING = license_expression.Licensing() + +collect_urls = re.compile('href="([^"]+)"').findall + +################################################################################ +# Fetch wheels and sources locally +################################################################################ + + +class DistributionNotFound(Exception): + pass + + +def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tuple()): + """ + Download the wheels binary distribution(s) of package ``name`` and + ``version`` matching the ``environment`` Environment constraints into the + ``dest_dir`` directory. Return a list of fetched_wheel_filenames, possibly + empty. + + Use the first PyPI simple repository from a list of ``repos`` that contains this wheel. + """ + if TRACE_DEEP: + print(f" download_wheel: {name}=={version} for envt: {environment}") + + if not repos: + repos = DEFAULT_PYPI_REPOS + + fetched_wheel_filenames = [] + + for repo in repos: + package = repo.get_package_version(name=name, version=version) + if not package: + if TRACE_DEEP: + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + continue + supported_wheels = list(package.get_supported_wheels(environment=environment)) + if not supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: No supported wheel for {name}=={version}: {environment} " + ) + continue + + for wheel in supported_wheels: + if TRACE_DEEP: + print( + f" download_wheel: Getting wheel from index (or cache): {wheel.download_url}" + ) + fetched_wheel_filename = wheel.download(dest_dir=dest_dir) + fetched_wheel_filenames.append(fetched_wheel_filename) + + if fetched_wheel_filenames: + # do not futher fetch from other repos if we find in first, typically PyPI + break + + return fetched_wheel_filenames + + +def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): + """ + Download the sdist source distribution of package ``name`` and ``version`` + into the ``dest_dir`` directory. Return a fetched filename or None. + + Use the first PyPI simple repository from a list of ``repos`` that contains + this sdist. + """ + if TRACE: + print(f" download_sdist: {name}=={version}") + + if not repos: + repos = DEFAULT_PYPI_REPOS + + fetched_sdist_filename = None + + for repo in repos: + package = repo.get_package_version(name=name, version=version) + + if not package: + if TRACE_DEEP: + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + continue + sdist = package.sdist + if not sdist: + if TRACE_DEEP: + print(f" download_sdist: No sdist for {name}=={version}") + continue + + if TRACE_DEEP: + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) + + if fetched_sdist_filename: + # do not futher fetch from other repos if we find in first, typically PyPI + break + + return fetched_sdist_filename + + +################################################################################ +# +# Core models +# +################################################################################ + + +@attr.attributes +class NameVer: + name = attr.ib( + type=str, + metadata=dict(help="Python package name, lowercase and normalized."), + ) + + version = attr.ib( + type=str, + metadata=dict(help="Python package version string."), + ) + + @property + def normalized_name(self): + return NameVer.normalize_name(self.name) + + @staticmethod + def normalize_name(name): + """ + Return a normalized package name per PEP503, and copied from + https://www.python.org/dev/peps/pep-0503/#id4 + """ + return name and re.sub(r"[-_.]+", "-", name).lower() or name + + def sortable_name_version(self): + """ + Return a tuple of values to sort by name, then version. + This method is a suitable to use as key for sorting NameVer instances. + """ + return self.normalized_name, packaging_version.parse(self.version) + + @classmethod + def sorted(cls, namevers): + return sorted(namevers or [], key=cls.sortable_name_version) + + +@attr.attributes +class Distribution(NameVer): + # field names that can be updated from another Distribution or mapping + updatable_fields = [ + "license_expression", + "copyright", + "description", + "homepage_url", + "primary_language", + "notice_text", + "extra_data", + ] + + filename = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="File name."), + ) + + path_or_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Path or URL"), + ) + + sha256 = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="SHA256 checksum."), + ) + + sha1 = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="SHA1 checksum."), + ) + + md5 = attr.ib( + repr=False, + type=int, + default=0, + metadata=dict(help="MD5 checksum."), + ) + + type = attr.ib( + repr=False, + type=str, + default="pypi", + metadata=dict(help="Package type"), + ) + + namespace = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Package URL namespace"), + ) + + qualifiers = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help="Package URL qualifiers"), + ) + + subpath = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Package URL subpath"), + ) + + size = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Size in bytes."), + ) + + primary_language = attr.ib( + repr=False, + type=str, + default="Python", + metadata=dict(help="Primary Programming language."), + ) + + description = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Description."), + ) + + homepage_url = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Homepage URL"), + ) + + notes = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Notes."), + ) + + copyright = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Copyright."), + ) + + license_expression = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="License expression"), + ) + + licenses = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help="List of license mappings."), + ) + + notice_text = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="Notice text"), + ) + + extra_data = attr.ib( + repr=False, + type=dict, + default=attr.Factory(dict), + metadata=dict(help="Extra data"), + ) + + @property + def package_url(self): + """ + Return a Package URL string of self. + """ + return str( + packageurl.PackageURL( + type=self.type, + namespace=self.namespace, + name=self.name, + version=self.version, + subpath=self.subpath, + qualifiers=self.qualifiers, + ) + ) + + @property + def download_url(self): + return self.get_best_download_url() + + def get_best_download_url(self, repos=tuple()): + """ + Return the best download URL for this distribution where best means this + is the first URL found for this distribution found in the list of + ``repos``. + + If none is found, return a synthetic PyPI remote URL. + """ + + if not repos: + repos = DEFAULT_PYPI_REPOS + + for repo in repos: + package = repo.get_package_version(name=self.name, version=self.version) + if not package: + if TRACE: + print( + f" get_best_download_url: {self.name}=={self.version} " + f"not found in {repo.index_url}" + ) + continue + pypi_url = package.get_url_for_filename(self.filename) + if pypi_url: + return pypi_url + else: + if TRACE: + print( + f" get_best_download_url: {self.filename} not found in {repo.index_url}" + ) + + def download(self, dest_dir=THIRDPARTY_DIR): + """ + Download this distribution into `dest_dir` directory. + Return the fetched filename. + """ + assert self.filename + if TRACE_DEEP: + print( + f"Fetching distribution of {self.name}=={self.version}:", + self.filename, + ) + + # FIXME: + fetch_and_save( + path_or_url=self.path_or_url, + dest_dir=dest_dir, + filename=self.filename, + as_text=False, + ) + return self.filename + + @property + def about_filename(self): + return f"{self.filename}.ABOUT" + + @property + def about_download_url(self): + return f"{ABOUT_BASE_URL}/{self.about_filename}" + + @property + def notice_filename(self): + return f"{self.filename}.NOTICE" + + @property + def notice_download_url(self): + return f"{ABOUT_BASE_URL}/{self.notice_filename}" + + @classmethod + def from_path_or_url(cls, path_or_url): + """ + Return a distribution built from the data found in the filename of a + ``path_or_url`` string. Raise an exception if this is not a valid + filename. + """ + filename = os.path.basename(path_or_url.strip("/")) + dist = cls.from_filename(filename) + dist.path_or_url = path_or_url + return dist + + @classmethod + def get_dist_class(cls, filename): + if filename.endswith(".whl"): + return Wheel + elif filename.endswith( + ( + ".zip", + ".tar.gz", + ) + ): + return Sdist + raise InvalidDistributionFilename(filename) + + @classmethod + def from_filename(cls, filename): + """ + Return a distribution built from the data found in a `filename` string. + Raise an exception if this is not a valid filename + """ + filename = os.path.basename(filename.strip("/")) + clazz = cls.get_dist_class(filename) + return clazz.from_filename(filename) + + def has_key_metadata(self): + """ + Return True if this distribution has key metadata required for basic attribution. + """ + if self.license_expression == "public-domain": + # copyright not needed + return True + return self.license_expression and self.copyright and self.path_or_url + + def to_about(self): + """ + Return a mapping of ABOUT data from this distribution fields. + """ + about_data = dict( + about_resource=self.filename, + checksum_md5=self.md5, + checksum_sha1=self.sha1, + copyright=self.copyright, + description=self.description, + download_url=self.download_url, + homepage_url=self.homepage_url, + license_expression=self.license_expression, + name=self.name, + namespace=self.namespace, + notes=self.notes, + notice_file=self.notice_filename if self.notice_text else "", + package_url=self.package_url, + primary_language=self.primary_language, + qualifiers=self.qualifiers, + size=self.size, + subpath=self.subpath, + type=self.type, + version=self.version, + ) + + about_data.update(self.extra_data) + about_data = {k: v for k, v in sorted(about_data.items()) if v} + return about_data + + def to_dict(self): + """ + Return a mapping data from this distribution. + """ + return {k: v for k, v in attr.asdict(self).items() if v} + + def save_about_and_notice_files(self, dest_dir=THIRDPARTY_DIR): + """ + Save a .ABOUT file to `dest_dir`. Include a .NOTICE file if there is a + notice_text. + """ + + def save_if_modified(location, content): + if os.path.exists(location): + with open(location) as fi: + existing_content = fi.read() + if existing_content == content: + return False + + if TRACE: + print(f"Saving ABOUT (and NOTICE) files for: {self}") + with open(location, "w") as fo: + fo.write(content) + return True + + as_about = self.to_about() + + save_if_modified( + location=os.path.join(dest_dir, self.about_filename), + content=saneyaml.dump(as_about), + ) + + notice_text = self.notice_text and self.notice_text.strip() + if notice_text: + save_if_modified( + location=os.path.join(dest_dir, self.notice_filename), + content=notice_text, + ) + + def load_about_data(self, about_filename_or_data=None, dest_dir=THIRDPARTY_DIR): + """ + Update self with ABOUT data loaded from an `about_filename_or_data` + which is either a .ABOUT file in `dest_dir` or an ABOUT data mapping. + `about_filename_or_data` defaults to this distribution default ABOUT + filename if not provided. Load the notice_text if present from dest_dir. + """ + if not about_filename_or_data: + about_filename_or_data = self.about_filename + + if isinstance(about_filename_or_data, str): + # that's an about_filename + about_path = os.path.join(dest_dir, about_filename_or_data) + if os.path.exists(about_path): + with open(about_path) as fi: + about_data = saneyaml.load(fi.read()) + if not about_data: + return False + else: + return False + else: + about_data = about_filename_or_data + + md5 = about_data.pop("checksum_md5", None) + if md5: + about_data["md5"] = md5 + sha1 = about_data.pop("checksum_sha1", None) + if sha1: + about_data["sha1"] = sha1 + sha256 = about_data.pop("checksum_sha256", None) + if sha256: + about_data["sha256"] = sha256 + + about_data.pop("about_resource", None) + notice_text = about_data.pop("notice_text", None) + notice_file = about_data.pop("notice_file", None) + if notice_text: + about_data["notice_text"] = notice_text + elif notice_file: + notice_loc = os.path.join(dest_dir, notice_file) + if os.path.exists(notice_loc): + with open(notice_loc) as fi: + about_data["notice_text"] = fi.read() + return self.update(about_data, keep_extra=True) + + def load_remote_about_data(self): + """ + Fetch and update self with "remote" data Distribution ABOUT file and + NOTICE file if any. Return True if the data was updated. + """ + try: + about_text = CACHE.get( + path_or_url=self.about_download_url, + as_text=True, + ) + except RemoteNotFetchedException: + return False + + if not about_text: + return False + + about_data = saneyaml.load(about_text) + notice_file = about_data.pop("notice_file", None) + if notice_file: + try: + notice_text = CACHE.get( + path_or_url=self.notice_download_url, + as_text=True, + ) + if notice_text: + about_data["notice_text"] = notice_text + except RemoteNotFetchedException: + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") + return self.load_about_data(about_data) + + def get_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return a mapping of computed checksums for this dist filename is + `dest_dir`. + """ + dist_loc = os.path.join(dest_dir, self.filename) + if os.path.exists(dist_loc): + return multi_checksums(dist_loc, checksum_names=("md5", "sha1", "sha256")) + else: + return {} + + def set_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with checksums computed for this dist filename is `dest_dir`. + """ + self.update(self.get_checksums(dest_dir), overwrite=True) + + def validate_checksums(self, dest_dir=THIRDPARTY_DIR): + """ + Return True if all checksums that have a value in this dist match + checksums computed for this dist filename is `dest_dir`. + """ + real_checksums = self.get_checksums(dest_dir) + for csk in ("md5", "sha1", "sha256"): + csv = getattr(self, csk) + rcv = real_checksums.get(csk) + if csv and rcv and csv != rcv: + return False + return True + + def get_license_keys(self): + try: + keys = LICENSING.license_keys( + self.license_expression, + unique=True, + simple=True, + ) + except license_expression.ExpressionParseError: + return ["unknown"] + return keys + + def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): + """ + Fetch license files if missing in `dest_dir`. + Return True if license files were fetched. + """ + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links + errors = [] + extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names += [self.extra_data.get("license_file")] + extra_lic_names = [ln for ln in extra_lic_names if ln] + lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] + for filename in lic_names + extra_lic_names: + floc = os.path.join(dest_dir, filename) + if os.path.exists(floc): + continue + + try: + # try remotely first + lic_url = get_license_link_for_filename(filename=filename, urls=urls) + + fetch_and_save( + path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, + as_text=True, + ) + if TRACE: + print(f"Fetched license from remote: {lic_url}") + + except: + try: + # try licensedb second + lic_url = f"{LICENSEDB_API_URL}/{filename}" + fetch_and_save( + path_or_url=lic_url, + dest_dir=dest_dir, + filename=filename, + as_text=True, + ) + if TRACE: + print(f"Fetched license from licensedb: {lic_url}") + + except: + msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + print(msg) + errors.append(msg) + + return errors + + def extract_pkginfo(self, dest_dir=THIRDPARTY_DIR): + """ + Return the text of the first PKG-INFO or METADATA file found in the + archive of this Distribution in `dest_dir`. Return None if not found. + """ + + fn = self.filename + if fn.endswith(".whl"): + fmt = "zip" + elif fn.endswith(".tar.gz"): + fmt = "gztar" + else: + fmt = None + + dist = os.path.join(dest_dir, fn) + with tempfile.TemporaryDirectory(prefix=f"pypi-tmp-extract-{fn}") as td: + shutil.unpack_archive(filename=dist, extract_dir=td, format=fmt) + # NOTE: we only care about the first one found in the dist + # which may not be 100% right + for pi in fileutils.resource_iter(location=td, with_dirs=False): + if pi.endswith( + ( + "PKG-INFO", + "METADATA", + ) + ): + with open(pi) as fi: + return fi.read() + + def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): + """ + Update self with data loaded from the PKG-INFO file found in the + archive of this Distribution in `dest_dir`. + """ + pkginfo_text = self.extract_pkginfo(dest_dir=dest_dir) + if not pkginfo_text: + print(f"!!!!PKG-INFO/METADATA not found in {self.filename}") + return + raw_data = email.message_from_string(pkginfo_text) + + classifiers = raw_data.get_all("Classifier") or [] + + declared_license = [raw_data["License"]] + [ + c for c in classifiers if c.startswith("License") + ] + license_expression = get_license_expression(declared_license) + other_classifiers = [c for c in classifiers if not c.startswith("License")] + + holder = raw_data["Author"] + holder_contact = raw_data["Author-email"] + copyright_statement = f"Copyright (c) {holder} <{holder_contact}>" + + pkginfo_data = dict( + name=raw_data["Name"], + declared_license=declared_license, + version=raw_data["Version"], + description=raw_data["Summary"], + homepage_url=raw_data["Home-page"], + copyright=copyright_statement, + license_expression=license_expression, + holder=holder, + holder_contact=holder_contact, + keywords=raw_data["Keywords"], + classifiers=other_classifiers, + ) + + return self.update(pkginfo_data, keep_extra=True) + + def update_from_other_dist(self, dist): + """ + Update self using data from another dist + """ + return self.update(dist.get_updatable_data()) + + def get_updatable_data(self, data=None): + data = data or self.to_dict() + return {k: v for k, v in data.items() if v and k in self.updatable_fields} + + def update(self, data, overwrite=False, keep_extra=True): + """ + Update self with a mapping of `data`. Keep unknown data as extra_data if + `keep_extra` is True. If `overwrite` is True, overwrite self with `data` + Return True if any data was updated, False otherwise. Raise an exception + if there are key data conflicts. + """ + package_url = data.get("package_url") + if package_url: + purl_from_data = packageurl.PackageURL.from_string(package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) + if purl_from_data != purl_from_self: + print( + f"Invalid dist update attempt, no same same purl with dist: " + f"{self} using data {data}." + ) + return + + data.pop("about_resource", None) + dl = data.pop("download_url", None) + if dl: + data["path_or_url"] = dl + + updated = False + extra = {} + for k, v in data.items(): + if isinstance(v, str): + v = v.strip() + if not v: + continue + + if hasattr(self, k): + value = getattr(self, k, None) + if not value or (overwrite and value != v): + try: + setattr(self, k, v) + except Exception as e: + raise Exception(f"{self}, {k}, {v}") from e + updated = True + + elif keep_extra: + # note that we always overwrite extra + extra[k] = v + updated = True + + self.extra_data.update(extra) + + return updated + + +def get_license_link_for_filename(filename, urls): + """ + Return a link for `filename` found in the `links` list of URLs or paths. Raise an + exception if no link is found or if there are more than one link for that + file name. + """ + path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + if not path_or_url: + raise Exception(f"Missing link to file: {filename}") + if not len(path_or_url) == 1: + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + return path_or_url[0] + + +class InvalidDistributionFilename(Exception): + pass + + +def get_sdist_name_ver_ext(filename): + """ + Return a (name, version, extension) if filename is a valid sdist name. Some legacy + binary builds have weird names. Return False otherwise. + + In particular they do not use PEP440 compliant versions and/or mix tags, os + and arch names in tarball names and versions: + + >>> assert get_sdist_name_ver_ext("intbitset-1.3.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-1.3.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-1.4a.zip") + >>> assert not get_sdist_name_ver_ext("intbitset-2.0.linux-x86_64.tar.gz") + >>> assert get_sdist_name_ver_ext("intbitset-2.0.tar.gz") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.src.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1-1.x86_64.rpm") + >>> assert not get_sdist_name_ver_ext("intbitset-2.1.linux-x86_64.tar.gz") + >>> assert not get_sdist_name_ver_ext("cffi-1.2.0-1.tar.gz") + >>> assert not get_sdist_name_ver_ext("html5lib-1.0-reupload.tar.gz") + >>> assert not get_sdist_name_ver_ext("selenium-2.0-dev-9429.tar.gz") + >>> assert not get_sdist_name_ver_ext("testfixtures-1.8.0dev-r4464.tar.gz") + """ + name_ver = None + extension = None + + for ext in EXTENSIONS_SDIST: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + return False + + name, _, version = name_ver.rpartition("-") + + if not name or not version: + return False + + # weird version + if any( + w in version + for w in ( + "x86_64", + "i386", + ) + ): + return False + + # all char versions + if version.isalpha(): + return False + + # non-pep 440 version + if "-" in version: + return False + + # single version + if version.isdigit() and len(version) == 1: + return False + + # r1 version + if len(version) == 2 and version[0] == "r" and version[1].isdigit(): + return False + + # dotless version (but calver is OK) + if "." not in version and len(version) < 3: + return False + + # version with dashes selenium-2.0-dev-9429.tar.gz + if name.endswith(("dev",)) and "." not in version: + return False + # version pre or post, old legacy + if version.startswith(("beta", "rc", "pre", "post", "final")): + return False + + return name, version, extension + + +@attr.attributes +class Sdist(Distribution): + extension = attr.ib( + repr=False, + type=str, + default="", + metadata=dict(help="File extension, including leading dot."), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a Sdist object built from a filename. + Raise an exception if this is not a valid sdist filename + """ + name_ver_ext = get_sdist_name_ver_ext(filename) + if not name_ver_ext: + raise InvalidDistributionFilename(filename) + + name, version, extension = name_ver_ext + + return cls( + type="pypi", + name=name, + version=version, + extension=extension, + filename=filename, + ) + + def to_filename(self): + """ + Return an sdist filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + return f"{self.name}-{self.version}.{self.extension}" + + +@attr.attributes +class Wheel(Distribution): + """ + Represents a wheel file. + + Copied and heavily modified from pip-20.3.1 copied from pip-20.3.1 + pip/_internal/models/wheel.py + + name: pip compatibility tags + version: 20.3.1 + download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py + copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + license_expression: mit + notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + + Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + """ + + get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, + ).match + + build = attr.ib( + type=str, + default="", + metadata=dict(help="Python wheel build."), + ) + + python_versions = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel Python version tags."), + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel ABI tags."), + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of wheel platform tags."), + ) + + tags = attr.ib( + repr=False, + type=set, + default=attr.Factory(set), + metadata=dict(help="Set of all tags for this wheel."), + ) + + @classmethod + def from_filename(cls, filename): + """ + Return a wheel object built from a filename. + Raise an exception if this is not a valid wheel filename + """ + wheel_info = cls.get_wheel_from_filename(filename) + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group("name").replace("_", "-") + # we'll assume "_" means "-" due to wheel naming scheme + # (https://github.com/pypa/pip/issues/1150) + version = wheel_info.group("ver").replace("_", "-") + build = wheel_info.group("build") + python_versions = wheel_info.group("pyvers").split(".") + abis = wheel_info.group("abis").split(".") + platforms = wheel_info.group("plats").split(".") + + # All the tag combinations from this file + tags = { + packaging_tags.Tag(x, y, z) for x in python_versions for y in abis for z in platforms + } + + return cls( + filename=filename, + type="pypi", + name=name, + version=version, + build=build, + python_versions=python_versions, + abis=abis, + platforms=platforms, + tags=tags, + ) + + def is_supported_by_tags(self, tags): + """ + Return True is this wheel is compatible with one of a list of PEP 425 tags. + """ + if TRACE_DEEP: + print() + print("is_supported_by_tags: tags:", tags) + print("self.tags:", self.tags) + return not self.tags.isdisjoint(tags) + + def to_filename(self): + """ + Return a wheel filename reconstructed from its fields (that may not be + the same as the original filename.) + """ + build = f"-{self.build}" if self.build else "" + pyvers = ".".join(self.python_versions) + abis = ".".join(self.abis) + plats = ".".join(self.platforms) + return f"{self.name}-{self.version}{build}-{pyvers}-{abis}-{plats}.whl" + + def is_pure(self): + """ + Return True if wheel `filename` is for a "pure" wheel e.g. a wheel that + runs on all Pythons 3 and all OSes. + + For example:: + + >>> Wheel.from_filename('aboutcode_toolkit-5.1.0-py2.py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py3-none-any.whl').is_pure() + True + >>> Wheel.from_filename('beautifulsoup4-4.7.1-py2-none-any.whl').is_pure() + False + >>> Wheel.from_filename('bitarray-0.8.1-cp36-cp36m-win_amd64.whl').is_pure() + False + >>> Wheel.from_filename('extractcode_7z-16.5-py2.py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-cp36-none-any.whl').is_pure() + False + >>> Wheel.from_filename('foo-4.7.1-py3-none-macosx_10_13_intel.whl').is_pure() + False + >>> Wheel.from_filename('future-0.16.0-py3-cp36m-any.whl').is_pure() + False + """ + return "py3" in self.python_versions and "none" in self.abis and "any" in self.platforms + + +def is_pure_wheel(filename): + try: + return Wheel.from_filename(filename).is_pure() + except: + return False + + +@attr.attributes +class PypiPackage(NameVer): + """ + A Python package contains one or more wheels and one source distribution + from a repository. + """ + + sdist = attr.ib( + repr=False, + type=Sdist, + default=None, + metadata=dict(help="Sdist source distribution for this package."), + ) + + wheels = attr.ib( + repr=False, + type=list, + default=attr.Factory(list), + metadata=dict(help="List of Wheel for this package"), + ) + + def get_supported_wheels(self, environment, verbose=TRACE_ULTRA_DEEP): + """ + Yield all the Wheel of this package supported and compatible with the + Environment `environment`. + """ + envt_tags = environment.tags() + if verbose: + print("get_supported_wheels: envt_tags:", envt_tags) + for wheel in self.wheels: + if wheel.is_supported_by_tags(envt_tags): + yield wheel + + @classmethod + def package_from_dists(cls, dists): + """ + Return a new PypiPackage built from an iterable of Wheels and Sdist + objects all for the same package name and version. + + For example: + >>> w1 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp38'], abis=['cp38m'], + ... platforms=['linux_x86_64']) + >>> w2 = Wheel(name='bitarray', version='0.8.1', build='', + ... python_versions=['cp38'], abis=['cp38m'], + ... platforms=['macosx_10_9_x86_64', 'macosx_10_10_x86_64']) + >>> sd = Sdist(name='bitarray', version='0.8.1') + >>> package = PypiPackage.package_from_dists(dists=[w1, w2, sd]) + >>> assert package.name == 'bitarray' + >>> assert package.version == '0.8.1' + >>> assert package.sdist == sd + >>> assert package.wheels == [w1, w2] + """ + dists = list(dists) + if TRACE_DEEP: + print(f"package_from_dists: {dists}") + if not dists: + return + + reference_dist = dists[0] + normalized_name = reference_dist.normalized_name + version = reference_dist.version + + package = PypiPackage(name=normalized_name, version=version) + + for dist in dists: + if dist.normalized_name != normalized_name: + if TRACE: + print( + f" Skipping inconsistent dist name: expected {normalized_name} got {dist}" + ) + continue + elif dist.version != version: + dv = packaging_version.parse(dist.version) + v = packaging_version.parse(version) + if dv != v: + if TRACE: + print( + f" Skipping inconsistent dist version: expected {version} got {dist}" + ) + continue + + if isinstance(dist, Sdist): + package.sdist = dist + + elif isinstance(dist, Wheel): + package.wheels.append(dist) + + else: + raise Exception(f"Unknown distribution type: {dist}") + + if TRACE_DEEP: + print(f"package_from_dists: {package}") + + return package + + @classmethod + def packages_from_dir(cls, directory): + """ + Yield PypiPackages built from files found in at directory path. + """ + base = os.path.abspath(directory) + + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + + if TRACE_ULTRA_DEEP: + print("packages_from_dir: paths:", paths) + return PypiPackage.packages_from_many_paths_or_urls(paths) + + @classmethod + def packages_from_many_paths_or_urls(cls, paths_or_urls): + """ + Yield PypiPackages built from a list of paths or URLs. + These are sorted by name and then by version from oldest to newest. + """ + dists = PypiPackage.dists_from_paths_or_urls(paths_or_urls) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls: dists:", dists) + + dists = NameVer.sorted(dists) + + for _projver, dists_of_package in itertools.groupby( + dists, + key=NameVer.sortable_name_version, + ): + package = PypiPackage.package_from_dists(dists_of_package) + if TRACE_ULTRA_DEEP: + print("packages_from_many_paths_or_urls", package) + yield package + + @classmethod + def dists_from_paths_or_urls(cls, paths_or_urls): + """ + Return a list of Distribution given a list of + ``paths_or_urls`` to wheels or source distributions. + + Each Distribution receives two extra attributes: + - the path_or_url it was created from + - its filename + + For example: + >>> paths_or_urls =''' + ... /home/foo/bitarray-0.8.1-cp36-cp36m-linux_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-macosx_10_9_x86_64.macosx_10_10_x86_64.whl + ... bitarray-0.8.1-cp36-cp36m-win_amd64.whl + ... https://example.com/bar/bitarray-0.8.1.tar.gz + ... bitarray-0.8.1.tar.gz.ABOUT + ... bit.LICENSE'''.split() + >>> results = list(PypiPackage.dists_from_paths_or_urls(paths_or_urls)) + >>> for r in results: + ... print(r.__class__.__name__, r.name, r.version) + ... if isinstance(r, Wheel): + ... print(" ", ", ".join(r.python_versions), ", ".join(r.platforms)) + Wheel bitarray 0.8.1 + cp36 linux_x86_64 + Wheel bitarray 0.8.1 + cp36 macosx_10_9_x86_64, macosx_10_10_x86_64 + Wheel bitarray 0.8.1 + cp36 win_amd64 + Sdist bitarray 0.8.1 + """ + dists = [] + if TRACE_ULTRA_DEEP: + print(" ###paths_or_urls:", paths_or_urls) + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] + for path_or_url in installable: + try: + dist = Distribution.from_path_or_url(path_or_url) + dists.append(dist) + if TRACE_DEEP: + print( + " ===> dists_from_paths_or_urls:", + dist, + "\n ", + "with URL:", + dist.download_url, + "\n ", + "from URL:", + path_or_url, + ) + except InvalidDistributionFilename: + if TRACE_DEEP: + print(f" Skipping invalid distribution from: {path_or_url}") + continue + return dists + + def get_distributions(self): + """ + Yield all distributions available for this PypiPackage + """ + if self.sdist: + yield self.sdist + for wheel in self.wheels: + yield wheel + + def get_url_for_filename(self, filename): + """ + Return the URL for this filename or None. + """ + for dist in self.get_distributions(): + if dist.filename == filename: + return dist.path_or_url + + +@attr.attributes +class Environment: + """ + An Environment describes a target installation environment with its + supported Python version, ABI, platform, implementation and related + attributes. + + We can use these to pass as `pip download` options and force fetching only + the subset of packages that match these Environment constraints as opposed + to the current running Python interpreter constraints. + """ + + python_version = attr.ib( + type=str, + default="", + metadata=dict(help="Python version supported by this environment."), + ) + + operating_system = attr.ib( + type=str, + default="", + metadata=dict(help="operating system supported by this environment."), + ) + + implementation = attr.ib( + type=str, + default="cp", + metadata=dict(help="Python implementation supported by this environment."), + repr=False, + ) + + abis = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of ABI tags supported by this environment."), + repr=False, + ) + + platforms = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of platform tags supported by this environment."), + repr=False, + ) + + @classmethod + def from_pyver_and_os(cls, python_version, operating_system): + if "." in python_version: + python_version = "".join(python_version.split(".")) + + return cls( + python_version=python_version, + implementation="cp", + abis=ABIS_BY_PYTHON_VERSION[python_version], + platforms=PLATFORMS_BY_OS[operating_system], + operating_system=operating_system, + ) + + def get_pip_cli_options(self): + """ + Return a list of pip download command line options for this environment. + """ + options = [ + "--python-version", + self.python_version, + "--implementation", + self.implementation, + ] + for abi in self.abis: + options.extend(["--abi", abi]) + + for platform in self.platforms: + options.extend(["--platform", platform]) + + return options + + def tags(self): + """ + Return a set of all the PEP425 tags supported by this environment. + """ + return set( + utils_pip_compatibility_tags.get_supported( + version=self.python_version or None, + impl=self.implementation or None, + platforms=self.platforms or None, + abis=self.abis or None, + ) + ) + + +################################################################################ +# +# PyPI repo and link index for package wheels and sources +# +################################################################################ + + +@attr.attributes +class PypiSimpleRepository: + """ + A PyPI repository of Python packages: wheels, sdist, etc. like the public + PyPI simple index. It is populated lazily based on requested packages names. + """ + + index_url = attr.ib( + type=str, + default=PYPI_SIMPLE_URL, + metadata=dict(help="Base PyPI simple URL for this index."), + ) + + # we keep a nested mapping of PypiPackage that has this shape: + # {name: {version: PypiPackage, version: PypiPackage, etc} + # the inner versions mapping is sorted by version from oldest to newest + + packages = attr.ib( + type=dict, + default=attr.Factory(lambda: defaultdict(dict)), + metadata=dict( + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + ), + ) + + fetched_package_normalized_names = attr.ib( + type=set, + default=attr.Factory(set), + metadata=dict(help="A set of already fetched package normalized names."), + ) + + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + ), + ) + + def _get_package_versions_map(self, name): + """ + Return a mapping of all available PypiPackage version for this package name. + The mapping may be empty. It is ordered by version from oldest to newest + """ + assert name + normalized_name = NameVer.normalize_name(name) + versions = self.packages[normalized_name] + if not versions and normalized_name not in self.fetched_package_normalized_names: + self.fetched_package_normalized_names.add(normalized_name) + try: + links = self.fetch_links(normalized_name=normalized_name) + # note that thsi is sorted so the mapping is also sorted + versions = { + package.version: package + for package in PypiPackage.packages_from_many_paths_or_urls(paths_or_urls=links) + } + self.packages[normalized_name] = versions + except RemoteNotFetchedException as e: + if TRACE: + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + + if not versions and TRACE: + print(f"WARNING: package {name} not found in repo: {self.index_url}") + + return versions + + def get_package_versions(self, name): + """ + Return a mapping of all available PypiPackage version as{version: + package} for this package name. The mapping may be empty but not None. + It is sorted by version from oldest to newest. + """ + return dict(self._get_package_versions_map(name)) + + def get_package_version(self, name, version=None): + """ + Return the PypiPackage with name and version or None. + Return the latest PypiPackage version if version is None. + """ + if not version: + versions = list(self._get_package_versions_map(name).values()) + # return the latest version + return versions and versions[-1] + else: + return self._get_package_versions_map(name).get(version) + + def fetch_links(self, normalized_name): + """ + Return a list of download link URLs found in a PyPI simple index for package + name using the `index_url` of this repository. + """ + package_url = f"{self.index_url}/{normalized_name}" + text = CACHE.get( + path_or_url=package_url, + as_text=True, + force=not self.use_cached_index, + ) + links = collect_urls(text) + # TODO: keep sha256 + links = [l.partition("#sha256=") for l in links] + links = [url for url, _, _sha256 in links] + return links + + +PYPI_PUBLIC_REPO = PypiSimpleRepository(index_url=PYPI_SIMPLE_URL) +PYPI_SELFHOSTED_REPO = PypiSimpleRepository(index_url=ABOUT_PYPI_SIMPLE_URL) +DEFAULT_PYPI_REPOS = PYPI_PUBLIC_REPO, PYPI_SELFHOSTED_REPO +DEFAULT_PYPI_REPOS_BY_URL = {r.index_url: r for r in DEFAULT_PYPI_REPOS} + + +@attr.attributes +class LinksRepository: + """ + Represents a simple links repository such an HTTP directory listing or an + HTML page with links. + """ + + url = attr.ib( + type=str, + default="", + metadata=dict(help="Links directory URL"), + ) + + links = attr.ib( + type=list, + default=attr.Factory(list), + metadata=dict(help="List of links available in this repo"), + ) + + use_cached_index = attr.ib( + type=bool, + default=False, + metadata=dict( + help="If True, use any existing on-disk cached index files. Otherwise, fetch and cache." + ), + ) + + def __attrs_post_init__(self): + if not self.links: + self.links = self.find_links() + + def find_links(self, _CACHE=[]): + """ + Return a list of link URLs found in the HTML page at `self.url` + """ + if _CACHE: + return _CACHE + + links_url = self.url + if TRACE_DEEP: + print(f"Finding links from: {links_url}") + plinks_url = urllib.parse.urlparse(links_url) + base_url = urllib.parse.SplitResult( + plinks_url.scheme, plinks_url.netloc, "", "", "" + ).geturl() + + if TRACE_DEEP: + print(f"Base URL {base_url}") + + text = CACHE.get( + path_or_url=links_url, + as_text=True, + force=not self.use_cached_index, + ) + + links = [] + for link in collect_urls(text): + if not link.endswith(EXTENSIONS): + continue + + plink = urllib.parse.urlsplit(link) + + if plink.scheme: + # full URL kept as-is + url = link + + if plink.path.startswith("/"): + # absolute link + url = f"{base_url}{link}" + + else: + # relative link + url = f"{links_url}/{link}" + + if TRACE_DEEP: + print(f"Adding URL: {url}") + + links.append(url) + + if TRACE: + print(f"Found {len(links)} links at {links_url}") + _CACHE.extend(links) + return links + + @classmethod + def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): + if url not in _LINKS_REPO: + _LINKS_REPO[url] = cls(url=url, use_cached_index=use_cached_index) + return _LINKS_REPO[url] + + +################################################################################ +# Globals for remote repos to be lazily created and cached on first use for the +# life of the session together with some convenience functions. +################################################################################ + + +def get_local_packages(directory=THIRDPARTY_DIR): + """ + Return the list of all PypiPackage objects built from a local directory. Return + an empty list if the package cannot be found. + """ + return list(PypiPackage.packages_from_dir(directory=directory)) + + +################################################################################ +# +# Basic file and URL-based operations using a persistent file-based Cache +# +################################################################################ + + +@attr.attributes +class Cache: + """ + A simple file-based cache based only on a filename presence. + This is used to avoid impolite fetching from remote locations. + """ + + directory = attr.ib(type=str, default=CACHE_THIRDPARTY_DIR) + + def __attrs_post_init__(self): + os.makedirs(self.directory, exist_ok=True) + + def get(self, path_or_url, as_text=True, force=False): + """ + Return the content fetched from a ``path_or_url`` through the cache. + Raise an Exception on errors. Treats the content as text if as_text is + True otherwise as treat as binary. `path_or_url` can be a path or a URL + to a file. + """ + cache_key = quote_plus(path_or_url.strip("/")) + cached = os.path.join(self.directory, cache_key) + + if force or not os.path.exists(cached): + if TRACE_DEEP: + print(f" FILE CACHE MISS: {path_or_url}") + content = get_file_content(path_or_url=path_or_url, as_text=as_text) + wmode = "w" if as_text else "wb" + with open(cached, wmode) as fo: + fo.write(content) + return content + else: + if TRACE_DEEP: + print(f" FILE CACHE HIT: {path_or_url}") + return get_local_file_content(path=cached, as_text=as_text) + + +CACHE = Cache() + + +def get_file_content(path_or_url, as_text=True): + """ + Fetch and return the content at `path_or_url` from either a local path or a + remote URL. Return the content as bytes is `as_text` is False. + """ + if path_or_url.startswith("https://"): + if TRACE_DEEP: + print(f"Fetching: {path_or_url}") + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + return content + + elif path_or_url.startswith("file://") or ( + path_or_url.startswith("/") and os.path.exists(path_or_url) + ): + return get_local_file_content(path=path_or_url, as_text=as_text) + + else: + raise Exception(f"Unsupported URL scheme: {path_or_url}") + + +def get_local_file_content(path, as_text=True): + """ + Return the content at `url` as text. Return the content as bytes is + `as_text` is False. + """ + if path.startswith("file://"): + path = path[7:] + + mode = "r" if as_text else "rb" + with open(path, mode) as fo: + return fo.read() + + +class RemoteNotFetchedException(Exception): + pass + + +def get_remote_file_content( + url, + as_text=True, + headers_only=False, + headers=None, + _delay=0, +): + """ + Fetch and return a tuple of (headers, content) at `url`. Return content as a + text string if `as_text` is True. Otherwise return the content as bytes. + + If `header_only` is True, return only (headers, None). Headers is a mapping + of HTTP headers. + Retries multiple times to fetch if there is a HTTP 429 throttling response + and this with an increasing delay. + """ + time.sleep(_delay) + headers = headers or {} + # using a GET with stream=True ensure we get the the final header from + # several redirects and that we can ignore content there. A HEAD request may + # not get us this last header + print(f" DOWNLOADING: {url}") + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + status = response.status_code + if status != requests.codes.ok: # NOQA + if status == 429 and _delay < 20: + # too many requests: start some exponential delay + increased_delay = (_delay * 2) or 1 + + return get_remote_file_content( + url, + as_text=as_text, + headers_only=headers_only, + _delay=increased_delay, + ) + + else: + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") + + if headers_only: + return response.headers, None + + return response.headers, response.text if as_text else response.content + + +def fetch_and_save( + path_or_url, + dest_dir, + filename, + as_text=True, +): + """ + Fetch content at ``path_or_url`` URL or path and save this to + ``dest_dir/filername``. Return the fetched content. Raise an Exception on + errors. Treats the content as text if as_text is True otherwise as treat as + binary. + """ + content = CACHE.get( + path_or_url=path_or_url, + as_text=as_text, + ) + output = os.path.join(dest_dir, filename) + wmode = "w" if as_text else "wb" + with open(output, wmode) as fo: + fo.write(content) + return content + + +################################################################################ +# +# Functions to update or fetch ABOUT and license files +# +################################################################################ + + +def clean_about_files( + dest_dir=THIRDPARTY_DIR, +): + """ + Given a thirdparty dir, clean ABOUT files + """ + local_packages = get_local_packages(directory=dest_dir) + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) + + if "classifiers" in local_dist.extra_data: + local_dist.extra_data.pop("classifiers", None) + local_dist.save_about_and_notice_files(dest_dir) + + +def fetch_abouts_and_licenses(dest_dir=THIRDPARTY_DIR, use_cached_index=False): + """ + Given a thirdparty dir, add missing ABOUT. LICENSE and NOTICE files using + best efforts: + + - use existing ABOUT files + - try to load existing remote ABOUT files + - derive from existing distribution with same name and latest version that + would have such ABOUT file + - extract ABOUT file data from distributions PKGINFO or METADATA files + + Use available existing on-disk cached index if use_cached_index is True. + """ + + def get_other_dists(_package, _dist): + """ + Return a list of all the dists from `_package` that are not the `_dist` + object + """ + return [d for d in _package.get_distributions() if d != _dist] + + local_packages = get_local_packages(directory=dest_dir) + packages_by_name = defaultdict(list) + for local_package in local_packages: + distributions = list(local_package.get_distributions()) + distribution = distributions[0] + packages_by_name[distribution.name].append(local_package) + + for local_package in local_packages: + for local_dist in local_package.get_distributions(): + local_dist.load_about_data(dest_dir=dest_dir) + local_dist.set_checksums(dest_dir=dest_dir) + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # lets try to get from another dist of the same local package + for otherd in get_other_dists(local_package, local_dist): + updated = local_dist.update_from_other_dist(otherd) + if updated and local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # try to get another version of the same package that is not our version + other_local_packages = [ + p + for p in packages_by_name[local_package.name] + if p.version != local_package.version + ] + other_local_version = other_local_packages and other_local_packages[-1] + if other_local_version: + latest_local_dists = list(other_local_version.get_distributions()) + for latest_local_dist in latest_local_dists: + latest_local_dist.load_about_data(dest_dir=dest_dir) + if not latest_local_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(latest_local_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) + continue + + # lets try to fetch remotely + local_dist.load_remote_about_data() + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + continue + + # try to get a latest version of the same package that is not our version + # and that is in our self hosted repo + lpv = local_package.version + lpn = local_package.name + + other_remote_packages = [ + p for v, p in PYPI_SELFHOSTED_REPO.get_package_versions(lpn).items() if v != lpv + ] + + latest_version = other_remote_packages and other_remote_packages[-1] + if latest_version: + latest_dists = list(latest_version.get_distributions()) + for remote_dist in latest_dists: + remote_dist.load_remote_about_data() + if not remote_dist.has_key_metadata(): + # there is not much value to get other data if we are missing the key ones + continue + else: + local_dist.update_from_other_dist(remote_dist) + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + break + + # if has key data we may look to improve later, but we can move on + if local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir=dest_dir) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index + ) + continue + + # try to get data from pkginfo (no license though) + local_dist.load_pkginfo_data(dest_dir=dest_dir) + + # FIXME: save as this is the last resort for now in all cases + # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): + local_dist.save_about_and_notice_files(dest_dir) + + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) + + if not local_dist.has_key_metadata(): + print(f"Unable to add essential ABOUT data for: {local_dist}") + if lic_errs: + lic_errs = "\n".join(lic_errs) + print(f"Failed to fetch some licenses:: {lic_errs}") + + +################################################################################ +# +# Functions to build new Python wheels including native on multiple OSes +# +################################################################################ + + +def call(args, verbose=TRACE): + """ + Call args in a subprocess and display output on the fly if ``trace`` is True. + Return a tuple of (returncode, stdout, stderr) + """ + if TRACE_DEEP: + print("Calling:", " ".join(args)) + with subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" + ) as process: + stdouts = [] + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + stdouts.append(line) + if verbose: + print(line.rstrip(), flush=True) + + stdout, stderr = process.communicate() + if not stdout.strip(): + stdout = "\n".join(stdouts) + return process.returncode, stdout, stderr + + +def download_wheels_with_pip( + requirements_specifiers=tuple(), + requirements_files=tuple(), + environment=None, + dest_dir=THIRDPARTY_DIR, + index_url=PYPI_SIMPLE_URL, + links_url=ABOUT_LINKS_URL, +): + """ + Fetch binary wheel(s) using pip for the ``envt`` Environment given a list of + pip ``requirements_files`` and a list of ``requirements_specifiers`` string + (such as package names or as name==version). + Return a tuple of (list of downloaded files, error string). + Do NOT fail on errors, but return an error message on failure. + """ + + cli_args = [ + "pip", + "download", + "--only-binary", + ":all:", + "--dest", + dest_dir, + "--index-url", + index_url, + "--find-links", + links_url, + "--no-color", + "--progress-bar", + "off", + "--no-deps", + "--no-build-isolation", + "--verbose", + # "--verbose", + ] + + if environment: + eopts = environment.get_pip_cli_options() + cli_args.extend(eopts) + else: + print("WARNING: no download environment provided.") + + cli_args.extend(requirements_specifiers) + for req_file in requirements_files: + cli_args.extend(["--requirement", req_file]) + + if TRACE: + print(f"Downloading wheels using command:", " ".join(cli_args)) + + existing = set(os.listdir(dest_dir)) + error = False + try: + returncode, _stdout, stderr = call(cli_args, verbose=True) + if returncode != 0: + error = stderr + except Exception as e: + error = str(e) + + if error: + print() + print("###########################################################################") + print("##################### Failed to fetch all wheels ##########################") + print("###########################################################################") + print(error) + print() + print("###########################################################################") + + downloaded = existing ^ set(os.listdir(dest_dir)) + return sorted(downloaded), error + + +################################################################################ +# +# Functions to check for problems +# +################################################################################ + + +def check_about(dest_dir=THIRDPARTY_DIR): + try: + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + except subprocess.CalledProcessError as cpe: + print() + print("Invalid ABOUT files:") + print(cpe.output.decode("utf-8", errors="replace")) + + +def find_problems( + dest_dir=THIRDPARTY_DIR, + report_missing_sources=False, + report_missing_wheels=False, +): + """ + Print the problems found in `dest_dir`. + """ + + local_packages = get_local_packages(directory=dest_dir) + + for package in local_packages: + if report_missing_sources and not package.sdist: + print(f"{package.name}=={package.version}: Missing source distribution.") + if report_missing_wheels and not package.wheels: + print(f"{package.name}=={package.version}: Missing wheels.") + + for dist in package.get_distributions(): + dist.load_about_data(dest_dir=dest_dir) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) + if not dist.has_key_metadata(): + print(f" Missing key ABOUT data in file://{abpth}") + if "classifiers" in dist.extra_data: + print(f" Dangling classifiers data in file://{abpth}") + if not dist.validate_checksums(dest_dir): + print(f" Invalid checksums in file://{abpth}") + if not dist.sha1 and dist.md5: + print(f" Missing checksums in file://{abpth}") + + check_about(dest_dir=dest_dir) + + +def get_license_expression(declared_licenses): + """ + Return a normalized license expression or None. + """ + if not declared_licenses: + return + try: + from packagedcode.licensing import get_only_expression_from_extracted_license + + return get_only_expression_from_extracted_license(declared_licenses) + except ImportError: + # Scancode is not installed, clean and join all the licenses + lics = [python_safe_name(l).lower() for l in declared_licenses] + return " AND ".join(lics).lower() diff --git a/etc/scripts/utils_thirdparty.py.ABOUT b/etc/scripts/utils_thirdparty.py.ABOUT new file mode 100644 index 00000000..84803494 --- /dev/null +++ b/etc/scripts/utils_thirdparty.py.ABOUT @@ -0,0 +1,15 @@ +about_resource: utils_thirdparty.py +package_url: pkg:github.com/pypa/pip/@20.3.1#src/pip/_internal/models/wheel.py +type: github +namespace: pypa +name: pip +version: 20.3.1 +subpath: src/pip/_internal/models/wheel.py + +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: copied from pip-20.3.1 pip/_internal/models/wheel.py + The models code has been heavily inspired from the ISC-licensed packaging-dists + https://github.com/uranusjr/packaging-dists by Tzu-ping Chung + \ No newline at end of file diff --git a/example/e2fsprogs-1.39/input/MAPPING.CONFIG b/example/e2fsprogs-1.39/input/MAPPING.CONFIG index e2815026..151ff050 100644 --- a/example/e2fsprogs-1.39/input/MAPPING.CONFIG +++ b/example/e2fsprogs-1.39/input/MAPPING.CONFIG @@ -14,20 +14,18 @@ # : # # Example: -# Assuming your input have columns "Resource", "Component", "Confirmed Version", "Confirmed Copyright", "Confirmed License" -# One should do: +# Assuming your input have columns "Resource", "Component" and wanted to convert "Resource" to "about_resource" +# and "Component" to "name" # -# about_file_path: Resource +# about_resource: Resource # name: Component # -# In addition, if there are fields that you want to put in the ABOUT -# files, please use the 'Custom Fields' to include these fields. # -# Note: All the Custom Field's keys will be converted to lower case and -# all the spaces will be replaced by '_' +# Note: All the Custom Field's keys will be converted to lower case and +# no spaces is allowed for the key's name. # # -# See http://www.dejacode.org/about_spec_v0.8.1.html for more information +# See https://aboutcode.readthedocs.io/projects/aboutcode-toolkit/ for more information # # Essential Fields diff --git a/example/e2fsprogs-1.39/input/e2fsprogs-1.39-INV.xlsx b/example/e2fsprogs-1.39/input/e2fsprogs-1.39-INV.xlsx index 4c2d684d..df2abf2b 100644 Binary files a/example/e2fsprogs-1.39/input/e2fsprogs-1.39-INV.xlsx and b/example/e2fsprogs-1.39/input/e2fsprogs-1.39-INV.xlsx differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..d3058aaf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,129 @@ +[build-system] +requires = ["setuptools>=70.0.0", "wheel", "setuptools_scm[tomm]>=8.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +# this is used populated when creating a git archive +# and when there is .git dir and/or there is no git installed +fallback_version = "9999.$Format:%h-%cs$" + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + "bin", + "dist", + "build", + "_build", + "etc", + "local", + "ci", + "docs", + "man", + "share", + "samples", + ".cache", + ".settings", + "Include", + "include", + "Lib", + "lib", + "lib64", + "Lib64", + "Scripts", + "thirdparty", + "tmp", + "venv", + ".venv", + "tests/data", + "*/tests/test_data", + ".eggs", + "src/*/data", + "tests/*/data" +] + +python_files = "*.py" + +python_classes = "Test" +python_functions = "test" + +addopts = [ + "-rfExXw", + "--strict-markers", + "--doctest-modules" +] + +[tool.ruff] +line-length = 100 +extend-exclude = [] +target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "tests/**/*.py", + "doc/**/*.py", + "docs/**/*.py", + "*.py", + "." + +] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +# django migrations + "**/migrations/**/*" +] + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ +# "E", # pycodestyle +# "W", # pycodestyle warnings + "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit + "I", # isort +# "C9", # McCabe complexity +] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] + + +[tool.ruff.lint.isort] +force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] + + +[tool.doc8] +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..87d1554f --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,49 @@ +alabaster==0.7.16 +anyio==4.6.0 +backports.tarfile==1.2.0 +babel==2.15.0 +doc8==1.1.2 +docutils==0.19 +execnet==1.9.0 +h11==0.16.0 +id==1.5.0 +imagesize==1.4.1 +iniconfig==1.1.1 +jaraco.classes==3.4.0 +jaraco.context==6.1.0 +jeepney==0.7.1 +keyring==23.4.1 +markdown-it-py==3.0.0 +mdurl==0.1.2 +nh3==0.2.18 +pytest==9.0.2 +pytest-xdist==2.5.0 +readme-renderer==34.0 +requests-toolbelt==0.9.1 +restructuredtext-lint==1.4.0 +rfc3986==1.5.0 +rich==12.3.0 +roman-numerals==3.1.0 +ruff==0.15.2 +secretstorage==3.3.2 +snowballstemmer==2.2.0 +Sphinx==7.2.6 +sphinx-autobuild==2024.2.4 +sphinx-copybutton==0.5.2 +sphinx-reredirects==0.1.3 +sphinx-rtd-dark-mode==1.3.0 +sphinx-rtd-theme==2.0.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-jquery==4.1 +sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-serializinghtml==1.1.10 +starlette==0.49.1 +stevedore==5.2.0 +twine==3.8.0 +typing-extensions==4.12.0 +uvicorn==0.30.6 +watchfiles==0.24.0 +websockets==14.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c5220184 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,28 @@ +attrs==25.4.0 +boolean.py==5.0 +certifi==2026.1.4 +cffi==2.0.0 +charset-normalizer==3.4.4 +click==8.3.1 +colorama==0.4.6 +commoncode==32.4.2 +cryptography==46.0.5 +et-xmlfile==2.0.0 +idna==3.11 +importlib-metadata==4.8.3 +jaraco.functools==4.4.0 +Jinja2==3.1.6 +license-expression==30.4.4 +MarkupSafe==3.0.3 +more-itertools==10.8.0 +openpyxl==3.1.5 +packageurl-python==0.17.6 +packaging==25.0 +pluggy==1.6.0 +pycparser==2.23 +Pygments==2.15.0 +PyYAML==6.0.3 +requests==2.32.5 +saneyaml==0.6.1 +urllib3==2.6.3 +zipp==3.23.0 diff --git a/setup.cfg b/setup.cfg index c29a174d..355101d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,47 +1,90 @@ -[bdist_wheel] -universal = 1 - [metadata] -license_file = NOTICE - -[aliases] -release = clean --all sdist --formats=bztar,zip bdist_wheel - -[tool:pytest] -norecursedirs = - .git - .cache - .settings - bin - dist - dist - build - _build - docs - man - share - example - etc - Include - include - Lib - lib - local - Scripts - thirdparty - tmp - tests/testdata/* - -python_files = *.py - -python_classes=Test -python_functions=test - -addopts = - -rfEsxXw - --strict - -s - -vv - --ignore docs/conf.py - --ignore setup.py - --doctest-modules +name = aboutcode-toolkit +license = Apache-2.0 + +# description must be on ONE line https://github.com/pypa/setuptools/issues/1390 +description = AboutCode-toolkit is a tool to document the provenance (origin and license) of third-party software using small text files. Collect inventories and generate attribution documentation. +long_description = file:README.rst +long_description_content_type = text/x-rst +url = https://github.com/aboutcode-org/aboutcode-toolkit + +author = nexB. Inc. and others +author_email = info@aboutcode.org + +classifiers = + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Topic :: Software Development + Topic :: Software Development :: Documentation + Topic :: Software Development :: Quality Assurance + Topic :: System :: Software Distribution + Topic :: Utilities + +keywords = + license + about + metadata + package + copyright + attribution + software + inventory + open source + sca + SBOM + spdx + +license_files = + apache-2.0.LICENSE + NOTICE + AUTHORS.rst + CHANGELOG.rst + CODE_OF_CONDUCT.rst + README.rst + +[options] +python_requires = >=3.9 + +package_dir = + =src +packages = find: +include_package_data = true +zip_safe = false + +install_requires = + attrs + boolean.py >= 3.5 + certifi + click + jinja2 + license_expression >= 0.94 + openpyxl + packageurl_python >= 0.9.0 + requests + saneyaml + + +[options.packages.find] +where = src + + +[options.extras_require] +dev = + pytest >= 7.0.1 + pytest-xdist >= 2 + twine + ruff + Sphinx>=5.0.2 + sphinx-rtd-theme>=1.0.0 + sphinx-reredirects >= 0.1.2 + doc8>=0.11.2 + sphinx-autobuild + sphinx-rtd-dark-mode>=1.3.0 + sphinx-copybutton +[options.entry_points] +console_scripts = + about=attributecode.cmd:about diff --git a/setup.py b/setup.py index a4046b90..bac24a43 100644 --- a/setup.py +++ b/setup.py @@ -1,91 +1,6 @@ #!/usr/bin/env python -# -*- encoding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function +import setuptools -from glob import glob -import io -from os.path import basename -from os.path import dirname -from os.path import join -from os.path import splitext - -from setuptools import find_packages -from setuptools import setup - - -def read(*names, **kwargs): - return io.open( - join(dirname(__file__), *names), - encoding=kwargs.get('encoding', 'utf8') - ).read() - - -setup( - name='aboutcode-toolkit', - version='4.0.2', - license='Apache-2.0', - description=( - 'AboutCode-toolkit is a tool to document the provenance (origin and license) of ' - 'third-party software using small text files. ' - 'Collect inventories, generate attribution documentation.' - ), - long_description=( - 'AttributeCode provides a simple way to document the ' - 'provenance (i.e. origin and license) of software components that ' - 'you use in your project. This documentation is stored in \*.ABOUT ' - 'files, side-by-side with the documented code.' - ), - author='Chin-Yeung Li, Jillian Daguil, Thomas Druez, Philippe Ombredanne and others.', - author_email='info@nexb.com', - url='http://aboutcode.org', - packages=find_packages('src'), - package_dir={'': 'src'}, - py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], - include_package_data=True, - zip_safe=False, - platforms='any', - classifiers=[ - 'Development Status :: 4 - Beta', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Topic :: Software Development', - 'Topic :: Software Development :: Documentation', - 'Topic :: Software Development :: Quality Assurance', - 'Topic :: System :: Software Distribution', - 'Topic :: Utilities', - ], - keywords=[ - 'license', 'about', 'metadata', 'package', 'copyright', - 'attribution', 'software', 'inventory', - ], - install_requires=[ - 'jinja2 >= 2.9, < 3.0', - - 'click >= 6.7, < 7.0', - - "backports.csv ; python_version<'3.6'", - - # required by saneyaml - 'PyYAML >= 3.11, <=3.13', - 'saneyaml', - - 'boolean.py >= 3.5, < 4.0', - 'license_expression >= 0.94', - ], - extras_require={ - ":python_version < '3.6'": ['backports.csv'], - }, - entry_points={ - 'console_scripts': [ - 'about=attributecode.cmd:about', - ] - }, -) +if __name__ == "__main__": + setuptools.setup() diff --git a/src/attributecode/__init__.py b/src/attributecode/__init__.py index 731868ba..5450d44d 100644 --- a/src/attributecode/__init__.py +++ b/src/attributecode/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,29 +14,18 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - from collections import namedtuple import logging import os -try: - # Python 2 - unicode # NOQA -except NameError: # pragma: nocover - # Python 3 - unicode = str # NOQA - import saneyaml -__version__ = '4.0.2' +__version__ = "11.1.1" -__about_spec_version__ = '3.1.4' +__about_spec_version__ = "4.0.0" __copyright__ = """ -Copyright (c) 2013-2020 nexB Inc. All rights reserved. http://dejacode.org +Copyright (c) nexB Inc. All rights reserved. http://dejacode.org Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -49,24 +38,24 @@ """ -class Error(namedtuple('Error', ['severity', 'message'])): +class Error(namedtuple("Error", ["severity", "message"])): """ An Error data with a severity and message. """ + def __new__(self, severity, message): if message: - if isinstance(message, unicode): + if isinstance(message, str): message = self._clean_string(message) else: - message = self._clean_string(unicode(repr(message), encoding='utf-8')) + message = self._clean_string(repr(message)) message = message.strip('"') - return super(Error, self).__new__( - Error, severity, message) + return super(Error, self).__new__(Error, severity, message) def __repr__(self, *args, **kwargs): sev, msg = self._get_values() - return 'Error(%(sev)s, %(msg)s)' % locals() + return "Error(%(sev)s, %(msg)s)" % locals() def __eq__(self, other): return repr(self) == repr(other) @@ -78,7 +67,7 @@ def _get_values(self): def render(self): sev, msg = self._get_values() - return '%(sev)s: %(msg)s' % locals() + return "%(sev)s: %(msg)s" % locals() def to_dict(self, *args, **kwargs): """ @@ -95,7 +84,7 @@ def _clean_string(s): if not s: return s if s.startswith(('u"', "u'")): - s = s.lstrip('u') + s = s.lstrip("u") s = s.replace('[u"', '["') s = s.replace("[u'", "['") s = s.replace("(u'", "('") @@ -116,12 +105,11 @@ def _clean_string(s): DEBUG = 10 NOTSET = 0 - severities = { - CRITICAL : 'CRITICAL', - ERROR : 'ERROR', - WARNING : 'WARNING', - INFO : 'INFO', - DEBUG : 'DEBUG', - NOTSET : 'NOTSET' + CRITICAL: "CRITICAL", + ERROR: "ERROR", + WARNING: "WARNING", + INFO: "INFO", + DEBUG: "DEBUG", + NOTSET: "NOTSET", } diff --git a/src/attributecode/__main__.py b/src/attributecode/__main__.py index 32a0ce70..bae1bc5f 100644 --- a/src/attributecode/__main__.py +++ b/src/attributecode/__main__.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,11 +14,7 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - - -if __name__ == '__main__': # pragma: nocover +if __name__ == "__main__": # pragma: nocover from attributecode import cmd + cmd.about() diff --git a/src/attributecode/api.py b/src/attributecode/api.py index 5eb4cef6..09579e42 100644 --- a/src/attributecode/api.py +++ b/src/attributecode/api.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,30 +14,15 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import json +from requests import get + +from urllib.parse import quote +from urllib.parse import urlencode +from urllib.error import HTTPError from attributecode import ERROR from attributecode import Error -from attributecode.util import python2 - - -if python2: # pragma: nocover - from urllib import quote # NOQA - from urllib import urlencode # NOQA - from urllib2 import HTTPError # NOQA - from urllib2 import Request # NOQA - from urllib2 import urlopen # NOQA -else: # pragma: nocover - from urllib.parse import quote # NOQA - from urllib.parse import urlencode # NOQA - from urllib.request import Request # NOQA - from urllib.request import urlopen # NOQA - from urllib.error import HTTPError # NOQA - """ API call helpers @@ -51,52 +36,39 @@ def request_license_data(api_url, api_key, license_key): `license_key`. Send a request to `api_url` authenticating with `api_key`. """ headers = { - 'Authorization': 'Token %s' % api_key, - } - payload = { - 'api_key': api_key, - 'key': license_key, - 'format': 'json' + "Authorization": "Token %s" % api_key, } + payload = {"api_key": api_key, "key": license_key, "format": "json"} - api_url = api_url.rstrip('/') + api_url = api_url.rstrip("/") payload = urlencode(payload) - full_url = '%(api_url)s/?%(payload)s' % locals() + full_url = "%(api_url)s/?%(payload)s" % locals() # handle special characters in URL such as space etc. quoted_url = quote(full_url, safe="%/:=&?~#+!$,;'@()*[]") license_data = {} errors = [] try: - request = Request(quoted_url, headers=headers) - response = urlopen(request) - response_content = response.read().decode('utf-8') + response = get(quoted_url, headers=headers) + response_content = response.text # FIXME: this should be an ordered dict license_data = json.loads(response_content) - if not license_data['results']: - msg = u"Invalid 'license': %s" % license_key + if not license_data.get("results", []): + msg = "Invalid 'license': %s" % license_key errors.append(Error(ERROR, msg)) - except HTTPError as http_e: - # some auth problem - if http_e.code == 403: - msg = (u"Authorization denied. Invalid '--api_key'. " - u"License generation is skipped.") - errors.append(Error(ERROR, msg)) - else: - # Since no api_url/api_key/network status have - # problem detected, it yields 'license' is the cause of - # this exception. - msg = u"Invalid 'license': %s" % license_key - errors.append(Error(ERROR, msg)) - + msg = "Authorization denied. Invalid '--api_key'. License generation is skipped." + errors.append(Error(ERROR, msg)) except Exception as e: - errors.append(Error(ERROR, str(e))) + # Already checked the authorization and accessible of the URL. + # The only exception left is URL is accessible, but it's not a valid API URL + msg = "Invalid '--api_url'. License generation is skipped." + errors.append(Error(ERROR, msg)) finally: - if license_data.get('count') == 1: - license_data = license_data.get('results')[0] + if license_data.get("count") == 1: + license_data = license_data.get("results")[0] else: license_data = {} @@ -113,7 +85,4 @@ def get_license_details_from_api(api_url, api_key, license_key): Missing values are provided as empty strings. """ license_data, errors = request_license_data(api_url, api_key, license_key) - license_name = license_data.get('name', '') - license_text = license_data.get('full_text', '') - license_key = license_data.get('key', '') - return license_name, license_key, license_text, errors + return license_data, errors diff --git a/src/attributecode/attrib.py b/src/attributecode/attrib.py index bfccca03..9f617b88 100644 --- a/src/attributecode/attrib.py +++ b/src/attributecode/attrib.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2020 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,13 +14,7 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import collections import datetime -import io import os import jinja2 @@ -31,113 +25,246 @@ from attributecode import Error from attributecode.licenses import COMMON_LICENSES from attributecode.model import parse_license_expression +from attributecode.model import License, StringField from attributecode.util import add_unc from attributecode.attrib_util import multi_sort - DEFAULT_TEMPLATE_FILE = os.path.join( - os.path.dirname(os.path.realpath(__file__)), '../../templates', 'default_html.template') + os.path.dirname(os.path.realpath(__file__)), "templates", "default_html.template" +) + +DEFAULT_TEMPLATE_SCANCODE_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "templates", "scancode_html.template" +) +DEFAULT_LICENSE_SCORE = 100 -def generate(abouts, template=None, variables=None): + +def generate( + abouts, is_about_input, license_dict, scancode, min_license_score, template=None, vartext=None +): """ Generate an attribution text from an `abouts` list of About objects, a - `template` template text and a `variables` optional dict of extra + `template` template text and a `vartext` optional dict of extra variables. Return a tuple of (error, attribution text) where error is an Error object or None and attribution text is the generated text or None. """ rendered = None - error = None + errors = [] template_error = check_template(template) if template_error: lineno, message = template_error error = Error( - CRITICAL, - 'Template validation error at line: {lineno}: "{message}"'.format(**locals()) + CRITICAL, 'Template validation error at line: {lineno}: "{message}"'.format(**locals()) ) + errors.append(error) return error, None template = jinja2.Template(template) + # Get the current UTC time + utcnow = datetime.datetime.utcnow() - try: - captured_license = [] - license_key_and_context = {} - sorted_license_key_and_context = {} - license_file_name_and_key = {} - license_key_to_license_name = {} - license_name_to_license_key = {} - # FIXME: This need to be simplified + licenses_list = [] + lic_name_expression_list = [] + if is_about_input: for about in abouts: - # about.license_file.value is a OrderDict with license_text_name as + # about.license_file.value is a OrderDict with license_file_name as # the key and the license text as the value - if about.license_file: - # We want to create a dictionary which have the license short name as - # the key and license text as the value - for license_text_name in about.license_file.value: - if not license_text_name in captured_license: - captured_license.append(license_text_name) - if license_text_name.endswith('.LICENSE'): - license_key = license_text_name.strip('.LICENSE') - else: - license_key = license_text_name - license_key_and_context[license_key] = about.license_file.value[license_text_name] - sorted_license_key_and_context = collections.OrderedDict(sorted(license_key_and_context.items())) - license_file_name_and_key[license_text_name] = license_key - - # Convert/map the key in license expression to license name - if about.license_expression.value and about.license_name.value: - special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) - lic_name_list = about.license_name.value - lic_name_expression_list = [] - - # The order of the license_name and key should be the same - # The length for both list should be the same - assert len(lic_name_list) == len(lic_list) - - # Map the license key to license name - index_for_license_name_list = 0 - for key in lic_list: - license_key_to_license_name[key] = lic_name_list[index_for_license_name_list] - license_name_to_license_key[lic_name_list[index_for_license_name_list]] = key - index_for_license_name_list = index_for_license_name_list + 1 - - # Create a license expression with license name instead of key - for segment in about.license_expression.value.split(): - if segment in license_key_to_license_name: - lic_name_expression_list.append(license_key_to_license_name[segment]) + index = 0 + for lic_name in about.license_name.value: + if about.license_key.value: + key = about.license_key.value[index] + else: + key = lic_name + captured = False + for lic in licenses_list: + if key in lic.key: + captured = True + break + if not captured or not licenses_list: + name = lic_name + if about.license_file.value.keys(): + filename = list(about.license_file.value.keys())[index] + text = list(about.license_file.value.values())[index] else: - lic_name_expression_list.append(segment) - - # Join the license name expression into a single string - lic_name_expression = ' '.join(lic_name_expression_list) - - # Add the license name expression string into the about object - about.license_name_expression = lic_name_expression - - # Get the current UTC time - utcnow = datetime.datetime.utcnow() - rendered = template.render( - abouts=abouts, common_licenses=COMMON_LICENSES, - license_key_and_context=sorted_license_key_and_context, - license_file_name_and_key=license_file_name_and_key, - license_key_to_license_name=license_key_to_license_name, - license_name_to_license_key=license_name_to_license_key, - utcnow=utcnow, - tkversion=__version__, - variables=variables - ) - except Exception as e: - lineno = getattr(e, 'lineno', '') or '' - if lineno: - lineno = ' at line: {}'.format(lineno) - err = getattr(e, 'message', '') or '' - error = Error( - CRITICAL, - 'Template processing error {lineno}: {err}'.format(**locals()), + error = Error(CRITICAL, "No license file found for " + name) + errors.append(error) + break + if about.license_url.value: + url = about.license_url.value[index] + else: + url = "" + license_object = License(key, name, filename, url, text) + licenses_list.append(license_object) + index = index + 1 + else: + # Create license object + for key in license_dict: + name = license_dict[key][0] + filename = license_dict[key][1] + text = license_dict[key][2] + url = license_dict[key][3] + license_object = License(key, name, filename, url, text) + licenses_list.append(license_object) + + # We need special treatment for scancode input. + # Each about_object may have duplicated license key and same/different license score + # We will only keep the unique license key with the highest license score. + # The process will update the license_key, license_name and license_score. + if scancode: + abouts, meet_score_licenses_list = generate_sctk_input( + abouts, min_license_score, license_dict ) - return error, rendered + # Remove the license object + remove_list = [] + for lic in licenses_list: + if lic.key not in meet_score_licenses_list: + remove_list.append(lic) + + for lic in remove_list: + licenses_list.remove(lic) + + for about in abouts: + # Create a license expression with license name + lic_name_expression = "" + lic_name_expression_list = [] + if about.license_expression.value: + for segment in about.license_expression.value.split(): + not_lic = True + for lic in licenses_list: + if segment == lic.key: + lic_name_expression_list.append(lic.name) + not_lic = False + break + if not_lic: + lic_name_expression_list.append(segment) + # Join the license name expression into a single string + lic_name_expression = " ".join(lic_name_expression_list) + + # Add the license name expression string into the about object as a custom field + custom_field = StringField( + name="license_name_expression", value=lic_name_expression, present=True + ) + setattr(about, "license_name_expression", custom_field) + + # Sort the about objects by name + abouts = sorted(abouts, key=lambda x: x.name.value.lower()) + + # Sort the license object by key + licenses_list = sorted(licenses_list, key=lambda x: x.key) + + rendered = template.render( + abouts=abouts, + common_licenses=COMMON_LICENSES, + licenses_list=licenses_list, + utcnow=utcnow, + tkversion=__version__, + vartext=vartext, + ) + + return errors, rendered + + +def generate_sctk_input(abouts, min_license_score, license_dict): + meet_score_licenses_list = [] + for about in abouts: + # We will use a dictionary to keep the unique license key + # which the dictionary key is the license key and the dictionary value + # is (lic_score, lic_name) + if about.license_key.value: + updated_dict = {} + lic_key = about.license_key.value + lic_name = [] + if about.license_name.value: + lic_name = about.license_name.value + else: + lic_name = [] + for key_list in lic_key: + lic_name_list = [] + for k in key_list: + try: + lic_name_list.append(license_dict[k][0]) + except: + lic_name_list.append(k) + lic_name.append(lic_name_list) + about.license_name.value = lic_name + + if not lic_name: + lic_name = [] + for key in lic_key: + lic_name.append(license_dict[key][0]) + lic_score = about.license_score.value + len_lic_key = len(lic_key) + if len_lic_key != len(lic_name): + raise ValueError( + f"Mismatch between lengths: lic_key ({len_lic_key}) vs lic_name ({len(lic_name)})" + ) + if len_lic_key != len(lic_score): + raise ValueError( + f"Mismatch between lengths: lic_key ({len_lic_key}) vs lic_score ({len(lic_score)})" + ) + lic_key_expression = about.license_key_expression.value + if lic_key_expression: + updated_lic_key_expression = [] + removed_index = [] + for index, key in enumerate(lic_key_expression): + if key in updated_dict: + previous_score, _name = updated_dict[key] + current_score = lic_score[index] + if current_score > previous_score: + updated_dict[key] = (lic_score[index], lic_name[index]) + # Track the duplicated index + removed_index.append(index) + else: + updated_dict[key] = (lic_score[index], lic_name[index]) + updated_lic_key_expression.append(key) + # Remove the duplication + for index, key in enumerate(about.license_key.value): + if index in removed_index: + del about.license_key.value[index] + del about.license_name.value[index] + del about.license_score.value[index] + + lic_key_expression = updated_lic_key_expression + updated_lic_key = [] + updated_lic_name = [] + updated_lic_score = [] + for index, lic in enumerate(updated_dict): + _sp_char, lic_keys, _invalid_lic_exp = parse_license_expression(lic) + score, name = updated_dict[lic] + if score >= min_license_score: + for lic_key in lic_keys: + if not lic_key in meet_score_licenses_list: + meet_score_licenses_list.append(lic_key) + + updated_lic_key.append(lic_keys) + updated_lic_name.append(name) + updated_lic_score.append(score) + + # Remove items that don't meet to score + for index, score in enumerate(updated_lic_score): + if score < min_license_score: + del updated_lic_key[index] + del updated_lic_name[index] + del updated_lic_score[index] + del lic_key_expression[index] + + about.license_key.value = updated_lic_key + about.license_name.value = updated_lic_name + about.license_score.value = updated_lic_score + about.license_key_expression.value = lic_key_expression + return abouts, meet_score_licenses_list + + +def get_license_file_key(license_text_name): + if license_text_name.endswith(".LICENSE"): + # See https://github.com/aboutcode-org/aboutcode-toolkit/issues/439 + # for why using split instead of strip + return license_text_name.rsplit(".", 1)[0] + else: + return license_text_name def check_template(template_string): @@ -146,60 +273,99 @@ def check_template(template_string): message) if the template is invalid or None if it is valid. """ try: - jinja2.filters.FILTERS['multi_sort'] = multi_sort + jinja2.filters.FILTERS["multi_sort"] = multi_sort jinja2.Template(template_string) except (jinja2.TemplateSyntaxError, jinja2.TemplateAssertionError) as e: return e.lineno, e.message -def generate_from_file(abouts, template_loc=DEFAULT_TEMPLATE_FILE, variables=None): +def generate_from_file( + abouts, + is_about_input, + license_dict, + scancode, + min_license_score, + template_loc=None, + vartext=None, +): """ Generate an attribution text from an `abouts` list of About objects, a - `template_loc` template file location and a `variables` optional + `template_loc` template file location and a `vartext` optional dict of extra variables. Return a tuple of (error, attribution text) where error is an Error object or None and attribution text is the generated text or None. """ - - template_loc = add_unc(template_loc) - with io.open(template_loc, encoding='utf-8') as tplf: + if not template_loc: + if scancode: + template_loc = add_unc(DEFAULT_TEMPLATE_SCANCODE_FILE) + else: + template_loc = add_unc(DEFAULT_TEMPLATE_FILE) + else: + template_loc = add_unc(template_loc) + with open(template_loc, encoding="utf-8", errors="replace") as tplf: tpls = tplf.read() - return generate(abouts, template=tpls, variables=variables) + return generate( + abouts, + is_about_input, + license_dict, + scancode, + min_license_score, + template=tpls, + vartext=vartext, + ) -def generate_and_save(abouts, output_location, template_loc=None, variables=None): +def generate_and_save( + abouts, + is_about_input, + license_dict, + output_location, + scancode=False, + min_license_score=0, + template_loc=None, + vartext=None, +): """ Generate an attribution text from an `abouts` list of About objects, a - `template_loc` template file location and a `variables` optional + `template_loc` template file location and a `vartext` optional dict of extra variables. Save the generated attribution text in the `output_location` file. Return a list of Error objects if any. """ errors = [] - # Parse license_expression and save to the license list for about in abouts: if not about.license_expression.value: continue - special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) - if special_char_in_expression: - msg = (u"The following character(s) cannot be in the license_expression: " + - str(special_char_in_expression)) + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + about.license_expression.value + ) + if special_char_in_expression or invalid_lic_exp: + if special_char_in_expression: + msg = "The following character(s) cannot be in the license_expression: " + str( + special_char_in_expression + ) + else: + msg = "This license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) rendering_error, rendered = generate_from_file( abouts, + is_about_input, + license_dict, + scancode=scancode, + min_license_score=min_license_score, template_loc=template_loc, - variables=variables + vartext=vartext, ) if rendering_error: - errors.append(rendering_error) + errors.extend(rendering_error) if rendered: output_location = add_unc(output_location) - with io.open(output_location, 'w', encoding='utf-8') as of: + with open(output_location, "w", encoding="utf-8", errors="replace") as of: of.write(rendered) - return errors + return errors, rendered diff --git a/src/attributecode/attrib_util.py b/src/attributecode/attrib_util.py index cf665905..9431076a 100644 --- a/src/attributecode/attrib_util.py +++ b/src/attributecode/attrib_util.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,17 +14,16 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - from jinja2 import Environment -from jinja2.filters import environmentfilter + +try: + from jinja2.filters import pass_environment +except ImportError: + from jinja2.filters import environmentfilter as pass_environment from jinja2.filters import make_attrgetter from jinja2.filters import ignore_case from jinja2.filters import FilterArgumentError - """ Extra JINJA2 custom filters and other template utilities. """ @@ -37,15 +36,12 @@ def get_template(template_text): """ env = Environment(autoescape=True) # register our custom filters - env.filters.update(dict( - unique_together=unique_together, - multi_sort=multi_sort)) + env.filters.update(dict(unique_together=unique_together, multi_sort=multi_sort)) return env.from_string(template_text) -@environmentfilter -def multi_sort(environment, value, reverse=False, case_sensitive=False, - attributes=None): +@pass_environment +def multi_sort(environment, value, reverse=False, case_sensitive=False, attributes=None): """ Sort an iterable using an "attributes" list of attribute names available on each iterable item. Sort ascending unless reverse is "true". Ignore the case @@ -59,9 +55,10 @@ def multi_sort(environment, value, reverse=False, case_sensitive=False, """ if not attributes: raise FilterArgumentError( - 'The multi_sort filter requires a list of attributes as argument, ' - 'such as in: ' - "for item in iterable|multi_sort(attributes=['date', 'name'])") + "The multi_sort filter requires a list of attributes as argument, " + "such as in: " + "for item in iterable|multi_sort(attributes=['date', 'name'])" + ) # build a list of attribute getters, one for each attribute do_ignore_case = ignore_case if not case_sensitive else None @@ -77,7 +74,7 @@ def key(v): return sorted(value, key=key, reverse=reverse) -@environmentfilter +@pass_environment def unique_together(environment, value, case_sensitive=False, attributes=None): """ Return a list of unique items from an iterable. Unicity is checked when @@ -93,9 +90,10 @@ def unique_together(environment, value, case_sensitive=False, attributes=None): """ if not attributes: raise FilterArgumentError( - 'The unique_together filter requires a list of attributes as argument, ' - 'such as in: ' - "{% for item in iterable|unique_together(attributes=['date', 'name']) %} ") + "The unique_together filter requires a list of attributes as argument, " + "such as in: " + "{% for item in iterable|unique_together(attributes=['date', 'name']) %} " + ) # build a list of attribute getters, one for each attribute do_ignore_case = ignore_case if not case_sensitive else None diff --git a/src/attributecode/cmd.py b/src/attributecode/cmd.py index 51daab7f..3f61b89c 100644 --- a/src/attributecode/cmd.py +++ b/src/attributecode/cmd.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,39 +14,51 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from attributecode.util import write_licenses +from attributecode.util import get_temp_dir +from attributecode.util import filter_errors +from attributecode.util import extract_zip +from attributecode.transform import Transformer +from attributecode.transform import write_excel +from attributecode.transform import write_json +from attributecode.transform import write_csv +from attributecode.transform import transform_excel +from attributecode.transform import transform_json +from attributecode.transform import transform_csv +from attributecode.transform import transform_data +from attributecode.model import write_output +from attributecode.model import pre_process_and_fetch_license_dict +from attributecode.model import get_copy_list +from attributecode.model import copy_redist_src +from attributecode.model import ( + collect_inventory, + collect_abouts_license_expression, + collect_inventory_license_expression, +) +from attributecode.gen import generate as generate_about_files, load_inventory +from attributecode.attrib import generate_and_save as generate_attribution_doc +from attributecode.attrib import DEFAULT_LICENSE_SCORE +from attributecode.attrib import check_template +from attributecode import severities +from attributecode import __version__ +from attributecode import __about_spec_version__ +from attributecode.util import unique +from attributecode import WARNING from collections import defaultdict from functools import partial -import io -import logging + import os import sys import click + # silence unicode literals warnings click.disable_unicode_literals_warning = True -from attributecode import WARNING -from attributecode.util import unique - -from attributecode import __about_spec_version__ -from attributecode import __version__ -from attributecode import severities -from attributecode.attrib import check_template -from attributecode.attrib import DEFAULT_TEMPLATE_FILE -from attributecode.attrib import generate_and_save as generate_attribution_doc -from attributecode.gen import generate as generate_about_files -from attributecode.model import collect_inventory -from attributecode.model import write_output -from attributecode.util import extract_zip -from attributecode.util import filter_errors - __copyright__ = """ - Copyright (c) 2013-2019 nexB Inc and others. All rights reserved. + Copyright (c) nexB Inc and others. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -57,47 +69,49 @@ See the License for the specific language governing permissions and limitations under the License.""" +prog_name = "AboutCode-toolkit" -prog_name = 'AboutCode-toolkit' - - -intro = '''%(prog_name)s version %(__version__)s +intro = """%(prog_name)s version %(__version__)s ABOUT spec version: %(__about_spec_version__)s https://aboutcode.org %(__copyright__)s -''' % locals() - +""" % locals() def print_version(): - click.echo('Running aboutcode-toolkit version ' + __version__) + click.echo("Running aboutcode-toolkit version " + __version__) class AboutCommand(click.Command): """ An enhanced click Command working around some Click quirk. """ - def main(self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra): + + def main(self, args=None, prog_name=None, complete_var=None, standalone_mode=True, **extra): """ Workaround click bug https://github.com/mitsuhiko/click/issues/365 """ return click.Command.main( - self, args=args, prog_name=self.name, - complete_var=complete_var, standalone_mode=standalone_mode, **extra) + self, + args=args, + prog_name=self.name, + complete_var=complete_var, + standalone_mode=standalone_mode, + **extra, + ) # we define a main entry command with subcommands -@click.group(name='about') +@click.group(name="about") @click.version_option(version=__version__, prog_name=prog_name, message=intro) -@click.help_option('-h', '--help') +@click.help_option("-h", "--help") def about(): """ -Generate licensing attribution and credit notices from .ABOUT files and inventories. + Generate licensing attribution and credit notices from .ABOUT files and inventories. -Read, write and collect provenance and license inventories from .ABOUT files to and from JSON or CSV files. + Read, write and collect provenance and license inventories from .ABOUT files to and from JSON or CSV files. -Use about --help for help on a command. + Use about --help for help on a command. """ @@ -105,6 +119,7 @@ def about(): # option validators ###################################################################### + def validate_key_values(ctx, param, value): """ Return the a dict of {key: value} if valid or raise a UsageError @@ -115,20 +130,30 @@ def validate_key_values(ctx, param, value): kvals, errors = parse_key_values(value) if errors: - ive = '\n'.join(sorted(' ' + x for x in errors)) - msg = ('Invalid {param} option(s):\n' - '{ive}'.format(**locals())) + ive = "\n".join(sorted(" " + x for x in errors)) + msg = "Invalid {param} option(s):\n{ive}".format(**locals()) raise click.UsageError(msg) return kvals -def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))): +def validate_extensions( + ctx, + param, + value, + extensions=tuple( + ( + ".csv", + ".json", + ) + ), +): if not value: return if not value.endswith(extensions): - msg = ' '.join(extensions) + msg = " ".join(extensions) raise click.UsageError( - 'Invalid {param} file extension: must be one of: {msg}'.format(**locals())) + "Invalid {param} file extension: must be one of: {msg}".format(**locals()) + ) return value @@ -136,58 +161,78 @@ def validate_extensions(ctx, param, value, extensions=tuple(('.csv', '.json',))) # inventory subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Collect the inventory of .ABOUT files to a CSV or JSON file.') - -@click.argument('location', - required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) -@click.argument('output', +@about.command( + cls=AboutCommand, + short_help="Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file or stdout.", +) +@click.argument( + "location", required=True, - metavar='OUTPUT', - type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) - -@click.option('-f', '--format', + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument("output", required=True, metavar="OUTPUT") +@click.option( + "--exclude", + multiple=True, + metavar="PATTERN", + help="Exclude the processing of the specified input pattern (e.g. *tests* or test/).", +) +@click.option( + "-f", + "--format", is_flag=False, - default='csv', + default="csv", show_default=True, - type=click.Choice(['json', 'csv']), - help='Set OUTPUT inventory file format.') - -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') - -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') - -@click.help_option('-h', '--help') - -def inventory(location, output, format, quiet, verbose): # NOQA + type=click.Choice(["json", "csv", "excel"]), + help="Set OUTPUT inventory file format.", +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def inventory(location, output, exclude, format, quiet, verbose): # NOQA """ -Collect the inventory of .ABOUT file data as CSV or JSON. + Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file. -LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. -OUTPUT: Path to the JSON or CSV inventory file to create. + OUTPUT: Path to the CSV/JSON/XLSX inventory file to create, or + using '-' to print result on screen/to stdout (Excel-formatted output + cannot be used in stdout). """ + # We are not using type=click.Path() to validate the output location as + # it does not support `-` , which is used to print the result to stdout. + if not output == "-": + parent_dir = os.path.dirname(output) + if not os.path.exists(parent_dir): + msg = "The OUTPUT directory: {parent_dir} does not exist.".format(**locals()) + msg += "\nPlease correct and re-run" + click.echo(msg) + sys.exit(1) + else: + # Check the format if output is stdout as xlsx format cannot be displayed. + if format == "excel": + msg = "Excel-formatted output cannot be used in stdout." + click.echo(msg) + sys.exit(0) if not quiet: print_version() - click.echo('Collecting inventory from ABOUT files...') + click.echo("Collecting inventory from ABOUT files...") - if location.lower().endswith('.zip'): + if location.lower().endswith(".zip"): # accept zipped ABOUT files as input location = extract_zip(location) - errors, abouts = collect_inventory(location) - write_errors = write_output(abouts=abouts, location=output, format=format) - errors.extend(write_errors) - errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') - if not quiet: - msg = 'Inventory collected in {output}.'.format(**locals()) + errors, abouts = collect_inventory(location, exclude) + write_output(abouts=abouts, location=output, format=format) + + if output == "-": + log_file_loc = None + else: + log_file_loc = output + "-error.log" + errors_count = report_errors(errors, quiet, verbose, log_file_loc) + if not quiet and not output == "-": + msg = "Inventory collected in {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) @@ -196,159 +241,517 @@ def inventory(location, output, format, quiet, verbose): # NOQA # gen subcommand ###################################################################### -@about.command(cls=AboutCommand, - short_help='Generate .ABOUT files from an inventory as CSV or JSON.') -@click.argument('location', +@about.command( + cls=AboutCommand, short_help="Generate .ABOUT files from an inventory as CSV/JSON/XLSX." +) +@click.argument( + "location", required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) - -@click.argument('output', + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", required=True, - metavar='OUTPUT', - type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True)) - + metavar="OUTPUT", + type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--android", + is_flag=True, + help="Generate MODULE_LICENSE_XXX (XXX will be replaced by license key) and NOTICE " + "as the same design as from Android.", +) # FIXME: the CLI UX should be improved with two separate options for API key and URL -@click.option('--fetch-license', +@click.option( + "--fetch-license", + is_flag=True, + help="Fetch license data and text files from the ScanCode LicenseDB.", +) +@click.option( + "--fetch-license-djc", nargs=2, type=str, - metavar='URL KEY', - help='Fetch license data and text files from a DejaCode License Library ' - 'API URL using the API KEY.') - -@click.option('--reference', - metavar='DIR', + metavar="api_url api_key", + help="Fetch license data and text files from a DejaCode License Library " + "API URL using the API KEY.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--reference", + metavar="DIR", type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True), - help='Path to a directory with reference license data and text files.') - -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') - -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') - -@click.help_option('-h', '--help') - -def gen(location, output, fetch_license, reference, quiet, verbose): + help="Path to a directory with reference license data and text files.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def gen( + location, + output, + android, + fetch_license, + fetch_license_djc, + scancode, + reference, + worksheet, + quiet, + verbose, +): """ -Generate .ABOUT files in OUTPUT from an inventory of .ABOUT files at LOCATION. + Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location. -LOCATION: Path to a JSON or CSV inventory file. + LOCATION: Path to a JSON/CSV/XLSX inventory file. -OUTPUT: Path to a directory where ABOUT files are generated. + OUTPUT: Path to a directory where ABOUT files are generated. """ if not quiet: print_version() - click.echo('Generating .ABOUT files...') + click.echo("Generating .ABOUT files...") - if not location.endswith(('.csv', '.json',)): - raise click.UsageError('ERROR: Invalid input file extension: must be one .csv or .json.') + # FIXME: This should be checked in the `click` + if not location.endswith((".csv", ".json", ".xlsx")): + raise click.UsageError( + "ERROR: Invalid input file extension: must be one .csv or .json or .xlsx." + ) + + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") errors, abouts = generate_about_files( location=location, base_dir=output, + android=android, reference_dir=reference, fetch_license=fetch_license, + fetch_license_djc=fetch_license_djc, + scancode=scancode, + worksheet=worksheet, ) - errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet: abouts_count = len(abouts) - msg = '{abouts_count} .ABOUT files generated in {output}.'.format(**locals()) + msg = "{abouts_count} .ABOUT files generated in {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) +###################################################################### +# gen_license subcommand +###################################################################### + + +@about.command( + cls=AboutCommand, + short_help="Fetch and save all the licenses in the license_expression field to a directory.", +) +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", + required=True, + metavar="OUTPUT", + type=click.Path(exists=True, file_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--djc", + nargs=2, + type=str, + metavar="api_url api_key", + help="Fetch licenses from a DejaCode License Library.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def gen_license(location, output, djc, scancode, worksheet, verbose): + """ + Fetch licenses (Default: ScanCode LicenseDB) in the license_expression field and save to the output location. + + LOCATION: Path to a JSON/CSV/XLSX/.ABOUT file(s) + + OUTPUT: Path to a directory where license files are saved. + """ + print_version() + api_url = "" + api_key = "" + errors = [] + + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") + + log_file_loc = os.path.join(output, "error.log") + + if location.endswith(".csv") or location.endswith(".json") or location.endswith(".xlsx"): + errors, abouts = collect_inventory_license_expression( + location=location, scancode=scancode, worksheet=worksheet + ) + if errors: + severe_errors_count = report_errors( + errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc + ) + sys.exit(severe_errors_count) + else: + # _errors, abouts = collect_inventory(location) + errors, abouts = collect_abouts_license_expression(location) + + if djc: + # Strip the ' and " for api_url, and api_key from input + api_url = djc[0].strip("'").strip('"') + api_key = djc[1].strip("'").strip('"') + + click.echo("Fetching licenses...") + from_check = False + license_dict, lic_errors = pre_process_and_fetch_license_dict( + abouts, from_check, api_url, api_key, scancode + ) + + if lic_errors: + errors.extend(lic_errors) + + # A dictionary with license file name as the key and context as the value + lic_dict_output = {} + for key in license_dict: + if not key in lic_dict_output: + lic_filename = license_dict[key][1] + lic_context = license_dict[key][2] + lic_dict_output[lic_filename] = lic_context + + write_errors = write_licenses(lic_dict_output, output) + if write_errors: + errors.extend(write_errors) + + severe_errors_count = report_errors( + errors, quiet=False, verbose=verbose, log_file_loc=log_file_loc + ) + sys.exit(severe_errors_count) + + ###################################################################### # attrib subcommand ###################################################################### + def validate_template(ctx, param, value): if not value: - return DEFAULT_TEMPLATE_FILE + return None - with io.open(value, encoding='utf-8') as templatef: + with open(value, encoding="utf-8", errors="replace") as templatef: template_error = check_template(templatef.read()) if template_error: lineno, message = template_error raise click.UsageError( - 'Template syntax error at line: ' - '{lineno}: "{message}"'.format(**locals())) + 'Template syntax error at line: {lineno}: "{message}"'.format(**locals()) + ) return value -@about.command(cls=AboutCommand, - short_help='Generate an attribution document from .ABOUT files.') - -@click.argument('location', +@about.command( + cls=AboutCommand, short_help="Generate an attribution document from JSON/CSV/XLSX/.ABOUT files." +) +@click.argument( + "input", required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) - -@click.argument('output', + metavar="INPUT", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument( + "output", required=True, - metavar='OUTPUT', - type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) - -@click.option('--template', - metavar='FILE', + metavar="OUTPUT", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), +) +@click.option( + "--api_url", nargs=1, type=click.STRING, metavar="URL", help="URL to DejaCode License Library." +) +@click.option( + "--api_key", + nargs=1, + type=click.STRING, + metavar="KEY", + help="API Key for the DejaCode License Library", +) +@click.option( + "--min-license-score", + type=int, + help="Attribute components that have license score higher than or equal to the defined " + "--min-license-score.", +) +@click.option( + "--scancode", is_flag=True, help="Indicate the input JSON file is from scancode_toolkit." +) +@click.option( + "--reference", + metavar="DIR", + type=click.Path(exists=True, file_okay=False, readable=True, resolve_path=True), + help='Path to a directory with reference files where "license_file" and/or "notice_file"' + " located.", +) +@click.option( + "--template", + metavar="FILE", callback=validate_template, type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), - help='Path to an optional custom attribution template to generate the ' - 'attribution document. If not provided the default built-in template is used.') - -@click.option('--vartext', + help="Path to an optional custom attribution template to generate the " + "attribution document. If not provided the default built-in template is used.", +) +@click.option( + "--vartext", multiple=True, callback=validate_key_values, - metavar='=', - help='Add variable text as key=value for use in a custom attribution template.') + metavar="=", + help="Add variable text as key=value for use in a custom attribution template.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def attrib( + input, + output, + api_url, + api_key, + scancode, + min_license_score, + reference, + template, + vartext, + worksheet, + quiet, + verbose, +): + """ + Generate an attribution document at OUTPUT using JSON, CSV or XLSX or .ABOUT files at INPUT. -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') + INPUT: Path to a file (.ABOUT/.csv/.json/.xlsx), directory or .zip archive containing .ABOUT files. -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') + OUTPUT: Path where to write the attribution document. + """ + # A variable to define if the input ABOUT file(s) + is_about_input = False + + rendered = "" + license_dict = {} + errors = [] + + if worksheet and not input.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") + + if not quiet: + print_version() + click.echo("Generating attribution...") + + # accept zipped ABOUT files as input + if input.lower().endswith(".zip"): + input = extract_zip(input) + + if scancode: + if not input.endswith(".json"): + msg = "The input file from scancode toolkit needs to be in JSON format." + click.echo(msg) + sys.exit(1) + if not min_license_score and not min_license_score == 0: + min_license_score = DEFAULT_LICENSE_SCORE + + if min_license_score: + if not scancode: + msg = ( + "This option requires a JSON file generated by scancode toolkit as the input. " + + 'The "--scancode" option is required.' + ) + click.echo(msg) + sys.exit(1) + + if input.endswith(".json") or input.endswith(".csv") or input.endswith(".xlsx"): + is_about_input = False + from_attrib = True + if not reference: + # Set current directory as the reference dir + reference = os.path.dirname(input) + # Since the errors from load_inventory is only about field formatting or + # empty field which is irrelevant for attribtion process, + # See https://github.com/nexB/aboutcode-toolkit/issues/524 + # I believe we do not need to capture these errors in attrib process + errors, abouts = load_inventory( + location=input, + from_attrib=from_attrib, + scancode=scancode, + reference_dir=reference, + worksheet=worksheet, + ) + + # Exit if CRITICAL error + if errors: + for e in errors: + if severities[e.severity] == "CRITICAL": + click.echo(e) + sys.exit(1) + + else: + is_about_input = True + _errors, abouts = collect_inventory(input) + + if not abouts: + msg = "No ABOUT file or reference is found from the input. Attribution generation halted." + click.echo(msg) + errors_count = 1 + sys.exit(errors_count) + + if not is_about_input: + # Check if both api_url and api_key present + if api_url or api_key: + if not api_url: + msg = '"--api_url" is required.' + click.echo(msg) + sys.exit(1) + if not api_key: + msg = '"--api_key" is required.' + click.echo(msg) + sys.exit(1) + else: + api_url = "" + api_key = "" + api_url = api_url.strip("'").strip('"') + api_key = api_key.strip("'").strip('"') + from_check = False + license_dict, lic_errors = pre_process_and_fetch_license_dict( + abouts, from_check, api_url, api_key, scancode, reference + ) + errors.extend(lic_errors) + sorted_license_dict = sorted(license_dict) + + # Read the license_file and store in a dictionary + for about in abouts: + if about.license_file.value or about.notice_file.value: + if not reference: + msg = '"license_file" / "notice_file" field contains value. Use `--reference` to indicate its parent directory.' + click.echo(msg) + # sys.exit(1) + + if abouts: + attrib_errors, rendered = generate_attribution_doc( + abouts=abouts, + is_about_input=is_about_input, + license_dict=dict(sorted(license_dict.items())), + output_location=output, + scancode=scancode, + min_license_score=min_license_score, + template_loc=template, + vartext=vartext, + ) + errors.extend(attrib_errors) + + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") + + if not quiet: + if rendered: + msg = "Attribution generated in: {output}".format(**locals()) + click.echo(msg) + else: + msg = "Attribution generation failed." + click.echo(msg) + sys.exit(errors_count) -@click.help_option('-h', '--help') -def attrib(location, output, template, vartext, quiet, verbose): +###################################################################### +# collect_redist_src subcommand +###################################################################### + + +@about.command(cls=AboutCommand, short_help="Collect redistributable sources.") +@click.argument( + "location", + required=True, + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.argument("output", required=True, metavar="OUTPUT") +@click.option( + "--from-inventory", + metavar="FILE", + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), + help="Path to an inventory CSV/JSON/XLSX file as the base list for files/directories " + "that need to be copied which have the 'redistribute' flagged.", +) +@click.option("--with-structures", is_flag=True, help="Copy sources with directory structure.") +@click.option("--zip", is_flag=True, help="Zip the copied sources to the output location.") +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def collect_redist_src(location, output, from_inventory, with_structures, zip, quiet, verbose): """ -Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. + Collect sources that have 'redistribute' flagged as 'True' in .ABOUT files or inventory + to the output location. -LOCATION: Path to a file, directory or .zip archive containing .ABOUT files. + LOCATION: Path to a directory containing sources that need to be copied + (and containing ABOUT files if `inventory` is not provided) -OUTPUT: Path where to write the attribution document. + OUTPUT: Path to a directory or a zip file where sources will be copied to. """ + if zip: + if not output.endswith(".zip"): + click.echo("The output needs to be a zip file.") + sys.exit() + if not quiet: print_version() - click.echo('Generating attribution...') + click.echo("Collecting inventory from ABOUT files...") - # accept zipped ABOUT files as input - if location.lower().endswith('.zip'): + if location.lower().endswith(".zip"): + # accept zipped ABOUT files as input location = extract_zip(location) - errors, abouts = collect_inventory(location) + if from_inventory: + errors, abouts = load_inventory(from_inventory, location) + else: + errors, abouts = collect_inventory(location) + + if zip: + # Copy to a temp location and the zip to the output location + output_location = get_temp_dir() + else: + output_location = output - attrib_errors = generate_attribution_doc( - abouts=abouts, - output_location=output, - template_loc=template, - variables=vartext, - ) - errors.extend(attrib_errors) + copy_list, copy_list_errors = get_copy_list(abouts, location) + copy_errors = copy_redist_src(copy_list, location, output_location, with_structures) - errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') + if zip: + import shutil + # Stripped the .zip extension as the `shutil.make_archive` will + # append the .zip extension + output_no_extension = output.rsplit(".", 1)[0] + shutil.make_archive(output_no_extension, "zip", output_location) + + errors.extend(copy_list_errors) + errors.extend(copy_errors) + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet: - msg = 'Attribution generated in: {output}'.format(**locals()) + msg = "Redistributed sources are copied to {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) @@ -359,32 +762,71 @@ def attrib(location, output, template, vartext, quiet, verbose): # FIXME: This is really only a dupe of the Inventory command -@about.command(cls=AboutCommand, - short_help='Validate that the format of .ABOUT files is correct and report ' - 'errors and warnings.') -@click.argument('location', +@about.command( + cls=AboutCommand, + short_help="Validate that the format of .ABOUT files is correct and report " + "errors and warnings.", +) +@click.argument( + "location", required=True, - metavar='LOCATION', - type=click.Path( - exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True)) - -@click.option('--verbose', - is_flag=True, - help='Show all error and warning messages.') - -@click.help_option('-h', '--help') - -def check(location, verbose): + metavar="LOCATION", + type=click.Path(exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True), +) +@click.option( + "--exclude", + multiple=True, + metavar="PATTERN", + help="Exclude the processing of the specified input pattern (e.g. *tests* or test/).", +) +@click.option("--license", is_flag=True, help="Validate the license_expression value in the input.") +@click.option( + "--djc", + nargs=2, + type=str, + metavar="api_url api_key", + help="Validate license_expression from a DejaCode License Library API URL using the API KEY.", +) +@click.option( + "--log", nargs=1, metavar="FILE", help="Path to a file to save the error messages if any." +) +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def check(location, exclude, license, djc, log, verbose): """ -Check .ABOUT file(s) at LOCATION for validity and print error messages. + Check .ABOUT file(s) at LOCATION for validity and print error messages. -LOCATION: Path to a file or directory containing .ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. """ print_version() - click.echo('Checking ABOUT files...') - errors, _abouts = collect_inventory(location) - severe_errors_count = report_errors(errors, quiet=False, verbose=verbose) + + if log: + # Check if the error log location exist and create the parent directory if not + parent = os.path.dirname(log) + if not parent: + os.makedirs(parent) + + api_url = "" + api_key = "" + if djc: + # Strip the ' and " for api_url, and api_key from input + api_url = djc[0].strip("'").strip('"') + api_key = djc[1].strip("'").strip('"') + click.echo("Checking ABOUT files...") + + errors, abouts = collect_inventory(location, exclude) + + # Validate license_expression + if license: + from_check = True + _key_text_dict, errs = pre_process_and_fetch_license_dict( + abouts, from_check, api_url, api_key + ) + for e in errs: + errors.append(e) + + severe_errors_count = report_errors(errors, quiet=False, verbose=verbose, log_file_loc=log) sys.exit(severe_errors_count) @@ -392,76 +834,127 @@ def check(location, verbose): # transform subcommand ###################################################################### + def print_config_help(ctx, param, value): if not value or ctx.resilient_parsing: return from attributecode.transform import tranformer_config_help + click.echo(tranformer_config_help) ctx.exit() -@about.command(cls=AboutCommand, - short_help='Transform a CSV by applying renamings, filters and checks.') - -@click.argument('location', +@about.command( + cls=AboutCommand, + short_help="Transform a CSV/JSON/XLSX by applying renamings, filters and checks.", +) +@click.argument( + "location", required=True, - callback=partial(validate_extensions, extensions=('.csv',)), - metavar='LOCATION', - type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True)) - -@click.argument('output', + callback=partial( + validate_extensions, + extensions=( + ".csv", + ".json", + ".xlsx", + ), + ), + metavar="LOCATION", + type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), +) +@click.argument( + "output", required=True, - callback=partial(validate_extensions, extensions=('.csv',)), - metavar='OUTPUT', - type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True)) - -@click.option('-c', '--configuration', - metavar='FILE', + callback=partial( + validate_extensions, + extensions=( + ".csv", + ".json", + ".xlsx", + ), + ), + metavar="OUTPUT", + type=click.Path(exists=False, dir_okay=False, writable=True, resolve_path=True), +) +@click.option( + "-c", + "--configuration", + metavar="FILE", type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True), - help='Path to an optional YAML configuration file. See --help-format for ' - 'format help.') - -@click.option('--help-format', - is_flag=True, is_eager=True, expose_value=False, - callback=print_config_help, - help='Show configuration file format help and exit.') - -@click.option('-q', '--quiet', - is_flag=True, - help='Do not print error or warning messages.') - -@click.option('--verbose', + help="Path to an optional YAML configuration file. See --help-format for format help.", +) +@click.option( + "--worksheet", + metavar="name", + help='The worksheet name from the INPUT. (Default: the "active" worksheet)', +) +@click.option( + "--help-format", is_flag=True, - help='Show all error and warning messages.') - -@click.help_option('-h', '--help') - -def transform(location, output, configuration, quiet, verbose): # NOQA + is_eager=True, + expose_value=False, + callback=print_config_help, + help="Show configuration file format help and exit.", +) +@click.option("-q", "--quiet", is_flag=True, help="Do not print error or warning messages.") +@click.option("--verbose", is_flag=True, help="Show all error and warning messages.") +@click.help_option("-h", "--help") +def transform(location, output, configuration, worksheet, quiet, verbose): # NOQA """ -Transform the CSV file at LOCATION by applying renamings, filters and checks -and write a new CSV to OUTPUT. + Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, filters and checks + and then write a new CSV/JSON/XLSX to OUTPUT. -LOCATION: Path to a CSV file. + LOCATION: Path to a CSV/JSON/XLSX file. -OUTPUT: Path to CSV inventory file to create. + OUTPUT: Path to CSV/JSON/XLSX inventory file to create. """ - from attributecode.transform import transform_csv_to_csv - from attributecode.transform import Transformer - - if not quiet: - print_version() - click.echo('Transforming CSV...') + if worksheet and not location.endswith(".xlsx"): + raise click.UsageError("ERROR: --worksheet option only works with .xlsx input.") if not configuration: transformer = Transformer.default() else: transformer = Transformer.from_file(configuration) - errors = transform_csv_to_csv(location, output, transformer) + if not transformer: + msg = "Cannot transform without Transformer" + click.echo(msg) + sys.exit(1) + + errors = [] + updated_data = [] + new_data = [] + + if location.endswith(".csv"): + new_data, errors = transform_csv(location) + elif location.endswith(".json"): + new_data, errors = transform_json(location) + elif location.endswith(".xlsx"): + new_data, errors = transform_excel(location, worksheet) + + if not errors: + updated_data, errors = transform_data(new_data, transformer) + + if not updated_data: + msg = "The input is empty. Nothing is transformed." + click.echo(msg) + sys.exit(0) + + if not errors: + if output.endswith(".csv"): + write_csv(output, updated_data) + elif output.endswith(".json"): + write_json(output, updated_data) + else: + write_excel(output, updated_data) - errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + '-error.log') + if not quiet: + print_version() + click.echo("Transforming...") + + errors_count = report_errors(errors, quiet, verbose, log_file_loc=output + "-error.log") if not quiet and not errors: - msg = 'Transformed CSV written to {output}.'.format(**locals()) + msg = "Transformed file is written to {output}.".format(**locals()) click.echo(msg) sys.exit(errors_count) @@ -470,6 +963,7 @@ def transform(location, output, configuration, quiet, verbose): # NOQA # Error management ###################################################################### + def report_errors(errors, quiet, verbose, log_file_loc=None): """ Report the `errors` list of Error objects to screen based on the `quiet` and @@ -479,47 +973,52 @@ def report_errors(errors, quiet, verbose, log_file_loc=None): file. Return True if there were severe error reported. """ - errors = unique(errors) - messages, severe_errors_count = get_error_messages(errors, quiet, verbose) - for msg in messages: - click.echo(msg) - if log_file_loc: - log_msgs, _ = get_error_messages(errors, quiet=False, verbose=True) - with io.open(log_file_loc, 'w', encoding='utf-8') as lf: - lf.write('\n'.join(log_msgs)) + severe_errors_count = 0 + if errors: + log_msgs, severe_errors_count = get_error_messages(errors, verbose) + if not quiet: + for msg in log_msgs: + click.echo(msg) + if log_msgs and log_file_loc: + with open(log_file_loc, "w", encoding="utf-8", errors="replace") as lf: + lf.write("\n".join(log_msgs)) + click.echo("Error log: " + log_file_loc) return severe_errors_count -def get_error_messages(errors, quiet=False, verbose=False): +def get_error_messages(errors, verbose=False): """ Return a tuple of (list of error message strings to report, severe_errors_count) given an `errors` list of Error objects and using the - `quiet` and `verbose` flags. + `verbose` flags. """ - errors = unique(errors) - severe_errors = filter_errors(errors, WARNING) + if verbose: + severe_errors = errors + else: + severe_errors = filter_errors(errors, WARNING) + + severe_errors = unique(severe_errors) severe_errors_count = len(severe_errors) messages = [] - if severe_errors and not quiet: - error_msg = 'Command completed with {} errors or warnings.'.format(severe_errors_count) + if severe_errors: + error_msg = "Command completed with {} errors or warnings.".format(severe_errors_count) messages.append(error_msg) - for severity, message in errors: - sevcode = severities.get(severity) or 'UNKNOWN' - msg = '{sevcode}: {message}'.format(**locals()) - if not quiet: - if verbose: - messages .append(msg) - elif severity >= WARNING: - messages .append(msg) + for severity, message in severe_errors: + sevcode = severities.get(severity) or "UNKNOWN" + msg = "{sevcode}: {message}".format(**locals()) + messages.append(msg) + return messages, severe_errors_count + ###################################################################### # Misc ###################################################################### + def parse_key_values(key_values): """ Given a list of "key=value" strings, return: @@ -533,7 +1032,7 @@ def parse_key_values(key_values): errors = set() parsed_key_values = defaultdict(list) for key_value in key_values: - key, _, value = key_value.partition('=') + key, _, value = key_value.partition("=") key = key.strip().lower() if not key: @@ -550,5 +1049,5 @@ def parse_key_values(key_values): return dict(parsed_key_values), sorted(errors) -if __name__ == '__main__': +if __name__ == "__main__": about() diff --git a/src/attributecode/gen.py b/src/attributecode/gen.py index e6266ffc..a8b2fa54 100644 --- a/src/attributecode/gen.py +++ b/src/attributecode/gen.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,14 +14,6 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -import codecs -from collections import OrderedDict - -# FIXME: why posipath??? from posixpath import basename from posixpath import dirname from posixpath import exists @@ -37,9 +29,11 @@ from attributecode.util import add_unc from attributecode.util import csv from attributecode.util import file_fields +from attributecode.util import invalid_chars from attributecode.util import to_posix from attributecode.util import UNC_PREFIX_POSIX -from attributecode.util import unique +from attributecode.util import load_scancode_json, load_csv, load_json, load_excel +from attributecode.util import strip_inventory_value def check_duplicated_columns(location): @@ -48,14 +42,13 @@ def check_duplicated_columns(location): at location. """ location = add_unc(location) - # FIXME: why errors=ignore? - with codecs.open(location, 'rb', encoding='utf-8', errors='ignore') as csvfile: + with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile: reader = csv.reader(csvfile) columns = next(reader) columns = [col for col in columns] seen = set() - dupes = OrderedDict() + dupes = dict() for col in columns: c = col.lower() if c in seen: @@ -69,54 +62,68 @@ def check_duplicated_columns(location): if dupes: dup_msg = [] for name, names in dupes.items(): - names = u', '.join(names) - msg = '%(name)s with %(names)s' % locals() + names = ", ".join(names) + msg = "%(name)s with %(names)s" % locals() dup_msg.append(msg) - dup_msg = u', '.join(dup_msg) - msg = ('Duplicated column name(s): %(dup_msg)s\n' % locals() + - 'Please correct the input and re-run.') - errors.append(Error(ERROR, msg)) - return unique(errors) + dup_msg = ", ".join(dup_msg) + msg = ( + "Duplicated column name(s): %(dup_msg)s\n" % locals() + + "Please correct the input and re-run." + ) + err = Error(ERROR, msg) + if not err in errors: + errors.append(err) + return errors -def check_duplicated_about_resource(inventory_dict): +def check_duplicated_about_resource(arp, arp_list): """ - Return a list of errors for duplicated about_resource in a CSV file at location. + Return error for duplicated about_resource. """ - arp_list = [] - errors = [] - for component in inventory_dict: - # Ignore all the empty path - if component['about_resource']: - if component['about_resource'] in arp_list: - msg = ("The input has duplicated values in 'about_resource' " - "field: " + component['about_resource']) - errors.append(Error(CRITICAL, msg)) - else: - arp_list.append(component['about_resource']) - return errors + if arp in arp_list: + msg = "The input has duplicated values in 'about_resource' field: " + arp + return Error(CRITICAL, msg) + return "" -def check_newline_in_file_field(inventory_dict): +def check_newline_in_file_field(component): """ Return a list of errors for newline characters detected in *_file fields. """ errors = [] - for component in inventory_dict: - for k in component.keys(): - if k in file_fields: - try: - if '\n' in component[k]: - msg = ("New line character detected in '%s' for '%s' which is not supported." - "\nPlease use ',' to declare multiple files.") % (k, component['about_resource']) - errors.append(Error(CRITICAL, msg)) - except: - pass + for k in component.keys(): + if k in file_fields: + try: + if "\n" in component[k]: + if k == "about_resource": + msg = ( + "Multiple lines detected in 'about_resource' for '%s' which is not supported." + ) % component["about_resource"] + else: + msg = ( + "New line character detected in '%s' for '%s' which is not supported." + "\nPlease use ',' to declare multiple files." + ) % (k, component["about_resource"]) + errors.append(Error(CRITICAL, msg)) + except: + pass return errors -# TODO: this should be either the CSV or the ABOUT files but not both??? -def load_inventory(location, base_dir, reference_dir=None): +def check_about_resource_filename(arp): + """ + Return error for invalid/non-support about_resource's filename or + empty string if no error is found. + """ + if invalid_chars(arp): + msg = "Invalid characters present in 'about_resource' field: " + arp + return Error(ERROR, msg) + return "" + + +def load_inventory( + location, from_attrib=False, base_dir=None, scancode=False, reference_dir=None, worksheet=None +): """ Load the inventory file at `location` for ABOUT and LICENSE files stored in the `base_dir`. Return a list of errors and a list of About objects @@ -127,105 +134,155 @@ def load_inventory(location, base_dir, reference_dir=None): """ errors = [] abouts = [] - base_dir = util.to_posix(base_dir) - # FIXME: do not mix up CSV and JSON - if location.endswith('.csv'): - # FIXME: this should not be done here. - dup_cols_err = check_duplicated_columns(location) - if dup_cols_err: - errors.extend(dup_cols_err) - return errors, abouts - inventory = util.load_csv(location) + is_spreadsheet = False + + if base_dir: + base_dir = util.to_posix(base_dir) + if scancode: + inventory = load_scancode_json(location) + else: + if location.endswith(".csv"): + dup_cols_err = check_duplicated_columns(location) + if dup_cols_err: + errors.extend(dup_cols_err) + return errors, abouts + inventory = load_csv(location) + is_spreadsheet = True + elif location.endswith(".xlsx"): + dup_cols_err, inventory = load_excel(location, worksheet) + is_spreadsheet = True + if dup_cols_err: + errors.extend(dup_cols_err) + return errors, abouts + else: + inventory = load_json(location) + + arp_list = [] + errors = [] + + if is_spreadsheet: + # Only the .csv and .xlsx may have newline issue + stripped_inv = strip_inventory_value(inventory) else: - inventory = util.load_json(location) - - try: - # FIXME: this should not be done here. - dup_about_resource_err = check_duplicated_about_resource(inventory) - if dup_about_resource_err: - errors.extend(dup_about_resource_err) - return errors, abouts - newline_in_file = check_newline_in_file_field(inventory) - if newline_in_file: - errors.extend(newline_in_file) - return errors, abouts - except Exception as e: - # TODO: why catch ALL Exception - msg = "The essential field 'about_resource' is not found in the " - errors.append(Error(CRITICAL, msg)) + stripped_inv = inventory + + for component in stripped_inv: + if not from_attrib: + if "about_resource" in component: + arp = component["about_resource"] + dup_err = check_duplicated_about_resource(arp, arp_list) + if dup_err: + if not dup_err in errors: + errors.append(dup_err) + else: + arp_list.append(arp) + + invalid_about_filename = check_about_resource_filename(arp) + if invalid_about_filename and not invalid_about_filename in errors: + errors.append(invalid_about_filename) + + newline_in_file_err = check_newline_in_file_field(component) + if newline_in_file_err: + errors.extend(newline_in_file_err) + + if errors: return errors, abouts - for i, fields in enumerate(inventory): + custom_fields_list = [] + for fields in stripped_inv: # check does the input contains the required fields required_fields = model.About.required_fields for f in required_fields: if f not in fields: - msg = "Required field: %(f)r not found in the " % locals() - errors.append(Error(ERROR, msg)) - return errors, abouts - afp = fields.get(model.About.ABOUT_RESOURCE_ATTR) - - # FIXME: this should not be a failure condition - if not afp or not afp.strip(): - msg = 'Empty column: %(afp)r. Cannot generate .ABOUT file.' % locals() - errors.append(Error(ERROR, msg)) - continue + if from_attrib and f == "about_resource": + continue + else: + msg = "Required field: %(f)r not found in the " % locals() + errors.append(Error(CRITICAL, msg)) + return errors, abouts + # Set about file path to '' if no 'about_resource' is provided from + # the input + if "about_resource" not in fields: + afp = "" else: - afp = util.to_posix(afp) + afp = fields.get(model.About.ABOUT_RESOURCE_ATTR) + + afp = util.to_posix(afp) + if base_dir: loc = join(base_dir, afp) + else: + loc = afp about = model.About(about_file_path=afp) about.location = loc # Update value for 'about_resource' # keep only the filename or '.' if it's a directory - if 'about_resource' in fields: - updated_resource_value = u'' - resource_path = fields['about_resource'] - if resource_path.endswith(u'/'): - updated_resource_value = u'.' + if "about_resource" in fields: + updated_resource_value = "" + resource_path = fields["about_resource"] + if resource_path.endswith("/"): + updated_resource_value = "." else: updated_resource_value = basename(resource_path) - fields['about_resource'] = updated_resource_value + fields["about_resource"] = updated_resource_value ld_errors = about.load_dict( fields, base_dir, + scancode=scancode, + from_attrib=from_attrib, running_inventory=False, reference_dir=reference_dir, ) - """ - # 'about_resource' field will be generated during the process. - # No error need to be raise for the missing 'about_resource'. - for e in ld_errors: - if e.message == 'Field about_resource is required': - ld_errors.remove(e) - """ - for e in ld_errors: - if not e in errors: - errors.extend(ld_errors) + + for severity, message in ld_errors: + if "Custom Field" in message: + field_name = message.replace("Custom Field: ", "").strip() + if not field_name in custom_fields_list: + custom_fields_list.append(field_name) + else: + errors.append(Error(severity, message)) + abouts.append(about) + if custom_fields_list: + custom_fields_err_msg = "Field " + str(custom_fields_list) + " is a custom field." + errors.append(Error(INFO, custom_fields_err_msg)) + + return errors, abouts - return unique(errors), abouts def update_about_resource(self): pass -def generate(location, base_dir, reference_dir=None, fetch_license=False): + +def generate( + location, + base_dir, + android=None, + reference_dir=None, + fetch_license=False, + fetch_license_djc=False, + scancode=False, + worksheet=None, +): """ Load ABOUT data from a CSV inventory at `location`. Write ABOUT files to base_dir. Return errors and about objects. """ - not_exist_errors = [] - api_url = '' - api_key = '' + notice_dict = {} + api_url = "" + api_key = "" gen_license = False # FIXME: use two different arguments: key and url # Check if the fetch_license contains valid argument - if fetch_license: + if fetch_license_djc: # Strip the ' and " for api_url, and api_key from input - api_url = fetch_license[0].strip("'").strip('"') - api_key = fetch_license[1].strip("'").strip('"') + api_url = fetch_license_djc[0].strip("'").strip('"') + api_key = fetch_license_djc[1].strip("'").strip('"') + gen_license = True + + if fetch_license: gen_license = True # TODO: WHY use posix?? @@ -234,11 +291,14 @@ def generate(location, base_dir, reference_dir=None, fetch_license=False): errors, abouts = load_inventory( location=location, base_dir=bdir, - reference_dir=reference_dir + reference_dir=reference_dir, + scancode=scancode, + worksheet=worksheet, ) - if gen_license: - license_dict, err = model.pre_process_and_fetch_license_dict(abouts, api_url, api_key) + license_dict, err = model.pre_process_and_fetch_license_dict( + abouts, api_url=api_url, api_key=api_key + ) if err: for e in err: # Avoid having same error multiple times @@ -246,19 +306,26 @@ def generate(location, base_dir, reference_dir=None, fetch_license=False): errors.append(e) for about in abouts: - if about.about_file_path.startswith('/'): - about.about_file_path = about.about_file_path.lstrip('/') - dump_loc = join(bdir, about.about_file_path.lstrip('/')) + # Strip trailing spaces + about.about_file_path = about.about_file_path.strip() + if about.about_file_path.startswith("/"): + about.about_file_path = about.about_file_path.lstrip("/") + # Use the name as the ABOUT file name if about_resource is empty + if not about.about_file_path: + about.about_file_path = about.name.value + dump_loc = join(bdir, about.about_file_path.lstrip("/")) # The following code is to check if there is any directory ends with spaces - split_path = about.about_file_path.split('/') + split_path = about.about_file_path.split("/") dir_endswith_space = False for segment in split_path: - if segment.endswith(' '): - msg = (u'File path : ' - u'%(dump_loc)s ' - u'contains directory name ends with spaces which is not ' - u'allowed. Generation skipped.' % locals()) + if segment.endswith(" "): + msg = ( + "File path : " + "%(dump_loc)s " + "contains directory name ends with spaces which is not " + "allowed. Generation skipped." % locals() + ) errors.append(Error(ERROR, msg)) dir_endswith_space = True break @@ -267,61 +334,73 @@ def generate(location, base_dir, reference_dir=None, fetch_license=False): continue try: - # Generate value for 'about_resource' if it does not exist - if not about.about_resource.value: - about.about_resource.value = OrderedDict() - about_resource_value = '' - if about.about_file_path.endswith('/'): - about_resource_value = u'.' - else: - about_resource_value = basename(about.about_file_path) - about.about_resource.value[about_resource_value] = None - about.about_resource.present = True - # Check for the existence of the 'about_resource' - # If the input already have the 'about_resource' field, it will - # be validated when creating the about object - loc = util.to_posix(dump_loc) - about_file_loc = loc - path = join(dirname(util.to_posix(about_file_loc)), about_resource_value) - if not exists(path): - path = util.to_posix(path.strip(UNC_PREFIX_POSIX)) - path = normpath(path) - msg = (u'Field about_resource: ' - u'%(path)s ' - u'does not exist' % locals()) - not_exist_errors.append(msg) - + licenses_dict = {} if gen_license: # Write generated LICENSE file license_key_name_context_url_list = about.dump_lic(dump_loc, license_dict) if license_key_name_context_url_list: - # use value not "presence" - if not about.license_file.present: - about.license_file.value = OrderedDict() - for lic_key, lic_name, lic_context, lic_url in license_key_name_context_url_list: - gen_license_name = lic_key + u'.LICENSE' - about.license_file.value[gen_license_name] = lic_context + for ( + lic_key, + lic_name, + lic_filename, + lic_context, + lic_url, + spdx_lic_key, + ) in license_key_name_context_url_list: + licenses_dict[lic_key] = [ + lic_name, + lic_filename, + lic_context, + lic_url, + spdx_lic_key, + ] + if not lic_name in about.license_name.value: + about.license_name.value.append(lic_name) + about.license_file.value[lic_filename] = lic_filename + if not lic_url in about.license_url.value: + about.license_url.value.append(lic_url) + if not spdx_lic_key in about.spdx_license_key.value: + about.spdx_license_key.value.append(spdx_lic_key) + if about.license_name.value: + about.license_name.present = True + if about.license_file.value: about.license_file.present = True - if not about.license_name.present: - about.license_name.value.append(lic_name) - if not about.license_url.present: - about.license_url.value.append(lic_url) if about.license_url.value: about.license_url.present = True - if about.license_name.value: - about.license_name.present = True + if about.spdx_license_key.value: + about.spdx_license_key.present = True + + about.dump(dump_loc, licenses_dict) - about.dump(dump_loc) + if android: + """ + Create MODULE_LICENSE_XXX and get context to create NOTICE file + follow the standard from Android Open Source Project + """ + import os - for e in not_exist_errors: - errors.append(Error(INFO, e)) + parent_path = os.path.dirname(util.to_posix(dump_loc)) + + about.android_module_license(parent_path) + notice_path, notice_context = about.android_notice(parent_path) + if notice_path in notice_dict.keys(): + notice_dict[notice_path] += "\n\n" + notice_context + else: + notice_dict[notice_path] = notice_context except Exception as e: # only keep the first 100 char of the exception # TODO: truncated errors are likely making diagnotics harder emsg = repr(e)[:100] - msg = (u'Failed to write .ABOUT file at : ' - u'%(dump_loc)s ' - u'with error: %(emsg)s' % locals()) + msg = "Failed to write .ABOUT file at : %(dump_loc)s with error: %(emsg)s" % locals() errors.append(Error(ERROR, msg)) - return unique(errors), abouts + + if android: + # Check if there is already a NOTICE file present + for path in notice_dict.keys(): + if os.path.exists(path): + msg = "NOTICE file already exist at: %s" % path + errors.append(Error(ERROR, msg)) + else: + about.dump_android_notice(path, notice_dict[path]) + return errors, abouts diff --git a/src/attributecode/licenses.py b/src/attributecode/licenses.py index 300fc8b6..393a6fc8 100644 --- a/src/attributecode/licenses.py +++ b/src/attributecode/licenses.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,76 +14,72 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - # Common license keys COMMON_LICENSES = ( - 'aes-128-3.0', - 'agpl-3.0-plus', - 'apache-1.1', - 'apache-2.0', - 'apple-attribution-1997', - 'apple-excl', - 'apsl-2.0', - 'arphic-public', - 'artistic-perl-1.0', - 'artistic-2.0', - 'bitstream', - 'boost-1.0', - 'broadcom-cfe', - 'bsd-new', - 'bsd-original', - 'bsd-original-uc', - 'bsd-simplified', - 'cmu-computing-services', - 'cddl-1.0', - 'cddl-1.1', - 'cpl-1.0', - 'cc-by-2.5', - 'cc-by-sa-3.0', - 'curl', - 'freetype', - 'gpl-1.0-plus', - 'gpl-2.0', - 'gpl-2.0-bison', - 'gpl-2.0-glibc', - 'gpl-2.0-plus', - 'gpl-3.0', - 'gpl-3.0-plus', - 'lgpl-2.0', - 'lgpl-2.0-plus', - 'lgpl-2.1', - 'lgpl-2.1-plus', - 'lgpl-3.0', - 'lgpl-3.0-plus', - 'gpl-2.0-plus-linking', - 'gpl-2.0-broadcom-linking', - 'ijg', - 'isc', - 'larabie', - 'libpng', - 'ms-limited-public', - 'ms-pl', - 'ms-rl', - 'ms-ttf-eula', - 'mit', - 'mpl-1.1', - 'mpl-2.0', - 'net-snmp', - 'npl-1.1', - 'ntpl', - 'openssl-ssleay', - 'ssleay-windows', - 'rsa-md4', - 'rsa-md5', - 'sfl-license', - 'sgi-freeb-2.0', - 'sun-rpc', - 'tcl', - 'tidy', - 'uoi-ncsa', - 'x11', - 'zlib', + "aes-128-3.0", + "agpl-3.0-plus", + "apache-1.1", + "apache-2.0", + "apple-attribution-1997", + "apple-excl", + "apsl-2.0", + "arphic-public", + "artistic-perl-1.0", + "artistic-2.0", + "bitstream", + "boost-1.0", + "broadcom-cfe", + "bsd-new", + "bsd-original", + "bsd-original-uc", + "bsd-simplified", + "cmu-computing-services", + "cddl-1.0", + "cddl-1.1", + "cpl-1.0", + "cc-by-2.5", + "cc-by-sa-3.0", + "curl", + "freetype", + "gpl-1.0-plus", + "gpl-2.0", + "gpl-2.0-bison", + "gpl-2.0-glibc", + "gpl-2.0-plus", + "gpl-3.0", + "gpl-3.0-plus", + "lgpl-2.0", + "lgpl-2.0-plus", + "lgpl-2.1", + "lgpl-2.1-plus", + "lgpl-3.0", + "lgpl-3.0-plus", + "gpl-2.0-plus-linking", + "gpl-2.0-broadcom-linking", + "ijg", + "isc", + "larabie", + "libpng", + "ms-limited-public", + "ms-pl", + "ms-rl", + "ms-ttf-eula", + "mit", + "mpl-1.1", + "mpl-2.0", + "net-snmp", + "npl-1.1", + "ntpl", + "openssl-ssleay", + "ssleay-windows", + "rsa-md4", + "rsa-md5", + "sfl-license", + "sgi-freeb-2.0", + "sun-rpc", + "tcl", + "tidy", + "uoi-ncsa", + "x11", + "zlib", ) diff --git a/src/attributecode/model.py b/src/attributecode/model.py index f0631999..66c19656 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -24,32 +24,19 @@ components inventories. """ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict -import io import json import os -# FIXME: why posixpath??? import posixpath +from requests import get, head, exceptions +import sys import traceback +from itertools import zip_longest -from attributecode.util import python2 - -if python2: # pragma: nocover - from itertools import izip_longest as zip_longest # NOQA - from urlparse import urljoin, urlparse # NOQA - from urllib2 import urlopen, Request, HTTPError # NOQA -else: # pragma: nocover - basestring = str # NOQA - from itertools import zip_longest # NOQA - from urllib.parse import urljoin, urlparse # NOQA - from urllib.request import urlopen, Request # NOQA - from urllib.error import HTTPError # NOQA +from urllib.parse import urljoin +from urllib.parse import urlparse from license_expression import Licensing +from packageurl import PackageURL from attributecode import __version__ from attributecode import CRITICAL @@ -59,23 +46,29 @@ from attributecode import api from attributecode import Error from attributecode import saneyaml +from attributecode import gen from attributecode import util +from attributecode.transform import write_excel from attributecode.util import add_unc from attributecode.util import boolean_fields from attributecode.util import copy_license_notice_files +from attributecode.util import copy_file from attributecode.util import csv from attributecode.util import file_fields from attributecode.util import filter_errors +from attributecode.util import get_spdx_key_and_lic_key_from_licdb from attributecode.util import is_valid_name from attributecode.util import on_windows +from attributecode.util import norm from attributecode.util import replace_tab_with_spaces from attributecode.util import wrap_boolean_value from attributecode.util import UNC_PREFIX from attributecode.util import ungroup_licenses -from attributecode.util import unique +from attributecode.util import ungroup_licenses_from_sctk genereated_tk_version = "# Generated with AboutCode Toolkit Version %s \n\n" % __version__ + class Field(object): """ An ABOUT file field. The initial value is a string. Subclasses can and @@ -86,10 +79,11 @@ def __init__(self, name=None, value=None, required=False, present=False): # normalized names are lowercased per specification self.name = name # save this and do not mutate it afterwards - if isinstance(value, basestring): + if isinstance(value, str): self.original_value = value - elif value: - self.original_value = repr(value) + # elif value: + # Don't convert it to string. Leave the format as-is + # self.original_value = repr(value) else: self.original_value = value @@ -103,7 +97,7 @@ def __init__(self, name=None, value=None, required=False, present=False): self.errors = [] def default_value(self): - return '' + return "" def validate(self, *args, **kwargs): """ @@ -116,7 +110,7 @@ def validate(self, *args, **kwargs): if not self.present: # required fields must be present if self.required: - msg = u'Field %(name)s is required' + msg = "Field %(name)s is required" errors.append(Error(CRITICAL, msg % locals())) return errors else: @@ -126,18 +120,17 @@ def validate(self, *args, **kwargs): if not name in boolean_fields and not self.has_content: # ... especially if required if self.required: - msg = u'Field %(name)s is required and empty' + msg = "Field %(name)s is required and empty" severity = CRITICAL else: severity = INFO - msg = u'Field %(name)s is present but empty.' + msg = "Field %(name)s is present but empty." errors.append(Error(severity, msg % locals())) else: # present fields with content go through validation... # first trim any trailing spaces on each line - if isinstance(self.original_value, basestring): - value = '\n'.join(s.rstrip() for s - in self.original_value.splitlines(False)) + if isinstance(self.original_value, str): + value = "\n".join(s.rstrip() for s in self.original_value.splitlines(False)) # then strip leading and trailing spaces value = value.strip() else: @@ -148,7 +141,7 @@ def validate(self, *args, **kwargs): errors.extend(validation_errors) except Exception as e: emsg = repr(e) - msg = u'Error validating field %(name)s: %(value)r: %(emsg)r' + msg = "Error validating field %(name)s: %(value)r: %(emsg)r" errors.append(Error(CRITICAL, msg % locals())) raise @@ -163,12 +156,15 @@ def _validate(self, *args, **kwargs): """ return [] + def _serialized_value(self): + return self.value if self.value else "" + def serialize(self): """ Return a unicode serialization of self in the ABOUT format. """ name = self.name - value = self.serialized_value() or u'' + value = self.serialized_value() or "" if self.has_content or self.value: value = value.splitlines(True) # multi-line @@ -176,29 +172,29 @@ def serialize(self): # This code is used to read the YAML's multi-line format in # ABOUT files # (Test: test_loads_dumps_is_idempotent) - if value[0].strip() == u'|' or value[0].strip() == u'>': - value = u' '.join(value) + if value[0].strip() == "|" or value[0].strip() == ">": + value = " ".join(value) else: # Insert '|' as the indicator for multi-line follow by a # newline character - value.insert(0, u'|\n') + value.insert(0, "|\n") # insert 4 spaces for newline values - value = u' '.join(value) + value = " ".join(value) else: # FIXME: See https://github.com/nexB/aboutcode-toolkit/issues/323 # The yaml.load() will throw error if the parsed value # contains ': ' character. A work around is to put a pipe, '|' # to indicate the whole value as a string - if value and ': ' in value[0]: - value.insert(0, u'|\n') + if value and ": " in value[0]: + value.insert(0, "|\n") # insert 4 spaces for newline values - value = u' '.join(value) + value = " ".join(value) else: - value = u''.join(value) + value = "".join(value) - serialized = u'%(name)s:' % locals() + serialized = "%(name)s:" % locals() if value: - serialized += ' ' + '%(value)s' % locals() + serialized += " " + "%(value)s" % locals() return serialized def serialized_value(self): @@ -206,7 +202,7 @@ def serialized_value(self): Return a unicode serialization of self in the ABOUT format. Does not include a white space for continuations. """ - return self._serialized_value() or u'' + return self._serialized_value() or "" @property def has_content(self): @@ -218,16 +214,18 @@ def __repr__(self): required = self.required has_content = self.has_content present = self.present - r = ('Field(name=%(name)r, value=%(value)r, required=%(required)r, present=%(present)r)') + r = "Field(name=%(name)r, value=%(value)r, required=%(required)r, present=%(present)r)" return r % locals() def __eq__(self, other): """ Equality based on string content value, ignoring spaces. """ - return (isinstance(other, self.__class__) - and self.name == other.name - and self.value == other.value) + return ( + isinstance(other, self.__class__) + and self.name == other.name + and self.value == other.value + ) class StringField(Field): @@ -235,19 +233,36 @@ class StringField(Field): A field containing a string value possibly on multiple lines. The validated value is a string. """ + def _validate(self, *args, **kwargs): - errors = super(StringField, self)._validate(*args, ** kwargs) + errors = super(StringField, self)._validate(*args, **kwargs) + no_special_char_field = [ + "license_expression", + "license_key", + "license_name", + "declared_license_expression", + "other_license_expression ", + ] + name = self.name + if name in no_special_char_field: + val = self.value + special_char = detect_special_char(val) + if special_char: + msg = ( + "The following character(s) cannot be in the %(name)s: " + "%(special_char)r" % locals() + ) + errors.append(Error(ERROR, msg)) return errors def _serialized_value(self): - return self.value if self.value else u'' + return self.value if self.value else "" def __eq__(self, other): """ Equality based on string content value, ignoring spaces """ - if not (isinstance(other, self.__class__) - and self.name == other.name): + if not (isinstance(other, self.__class__) and self.name == other.name): return False if self.value == other.value: @@ -255,12 +270,12 @@ def __eq__(self, other): # compare values stripped from spaces. Empty and None are equal if self.value: - sval = u''.join(self.value.split()) + sval = "".join(self.value.split()) if not sval: sval = None if other.value: - oval = u''.join(other.value.split()) + oval = "".join(other.value.split()) if not oval: oval = None @@ -273,13 +288,13 @@ class SingleLineField(StringField): A field containing a string value on a single line. The validated value is a string. """ + def _validate(self, *args, **kwargs): - errors = super(SingleLineField, self)._validate(*args, ** kwargs) - if self.value and isinstance(self.value, basestring) and '\n' in self.value: + errors = super(SingleLineField, self)._validate(*args, **kwargs) + if self.value and isinstance(self.value, str) and "\n" in self.value: name = self.name value = self.original_value - msg = (u'Field %(name)s: Cannot span multiple lines: %(value)s' - % locals()) + msg = "Field %(name)s: Cannot span multiple lines: %(value)s" % locals() errors.append(Error(ERROR, msg)) return errors @@ -289,16 +304,17 @@ class ListField(StringField): A field containing a list of string values, one per line. The validated value is a list. """ + def default_value(self): return [] def _validate(self, *args, **kwargs): - errors = super(ListField, self)._validate(*args, ** kwargs) + errors = super(ListField, self)._validate(*args, **kwargs) # reset self.value = [] - if isinstance(self.original_value, basestring): + if isinstance(self.original_value, str): values = self.original_value.splitlines(False) elif isinstance(self.original_value, list): values = self.original_value @@ -306,12 +322,11 @@ def _validate(self, *args, **kwargs): values = [repr(self.original_value)] for val in values: - if isinstance(val, basestring): + if isinstance(val, str): val = val.strip() if not val: name = self.name - msg = (u'Field %(name)s: ignored empty list value' - % locals()) + msg = "Field %(name)s: ignored empty list value" % locals() errors.append(Error(INFO, msg)) continue # keep only unique and report error for duplicates @@ -319,21 +334,19 @@ def _validate(self, *args, **kwargs): self.value.append(val) else: name = self.name - msg = (u'Field %(name)s: ignored duplicated list value: ' - '%(val)r' % locals()) + msg = "Field %(name)s: ignored duplicated list value: %(val)r" % locals() errors.append(Error(WARNING, msg)) return errors def _serialized_value(self): - return self.value if self.value else u'' + return self.value if self.value else "" def __eq__(self, other): """ Equality based on sort-insensitive values """ - if not (isinstance(other, self.__class__) - and self.name == other.name): + if not (isinstance(other, self.__class__) and self.name == other.name): return False if self.value == other.value: @@ -351,20 +364,50 @@ def __eq__(self, other): if sval == oval: return True + +class PackageUrlField(StringField): + """ + A Package URL field. The validated value is a purl. + """ + + def _validate(self, *args, **kwargs): + """ + Check that Package URL is valid. Return a list of errors. + """ + errors = super(PackageUrlField, self)._validate(*args, **kwargs) + name = self.name + val = self.value + if not self.is_valid_purl(val): + msg = "Field %(name)s: Invalid Package URL: %(val)s" % locals() + errors.append(Error(WARNING, msg)) + return errors + + @staticmethod + def is_valid_purl(purl): + """ + Return True if a Package URL is valid. + """ + try: + return bool(PackageURL.from_string(purl)) + except: + return False + + class UrlListField(ListField): """ A URL field. The validated value is a list of URLs. """ + def _validate(self, *args, **kwargs): """ Check that URLs are valid. Return a list of errors. """ - errors = super(UrlListField, self)._validate(*args, ** kwargs) + errors = super(UrlListField, self)._validate(*args, **kwargs) name = self.name val = self.value for url in val: if not self.is_valid_url(url): - msg = (u'Field %(name)s: Invalid URL: %(val)s' % locals()) + msg = "Field %(name)s: Invalid URL: %(val)s" % locals() errors.append(Error(WARNING, msg)) return errors @@ -374,7 +417,7 @@ def is_valid_url(url): Return True if a URL is valid. """ scheme, netloc, _path, _p, _q, _frg = urlparse(url) - valid = scheme in ('http', 'https', 'ftp') and netloc + valid = scheme in ("http", "https", "ftp") and netloc return valid @@ -382,15 +425,16 @@ class UrlField(StringField): """ A URL field. The validated value is a URL. """ + def _validate(self, *args, **kwargs): """ Check that URL is valid. Return a list of errors. """ - errors = super(UrlField, self)._validate(*args, ** kwargs) + errors = super(UrlField, self)._validate(*args, **kwargs) name = self.name val = self.value if not self.is_valid_url(val): - msg = (u'Field %(name)s: Invalid URL: %(val)s' % locals()) + msg = "Field %(name)s: Invalid URL: %(val)s" % locals() errors.append(Error(WARNING, msg)) return errors @@ -400,7 +444,7 @@ def is_valid_url(url): Return True if a URL is valid. """ scheme, netloc, _path, _p, _q, _frg = urlparse(url) - valid = scheme in ('http', 'https', 'ftp') and netloc + valid = scheme in ("http", "https", "ftp") and netloc return valid @@ -410,6 +454,7 @@ class PathField(ListField): The validated value is an ordered dict of path->location or None. The paths can also be resolved """ + def default_value(self): return {} @@ -421,26 +466,26 @@ def _validate(self, *args, **kwargs): base_dir is the directory location of the ABOUT file used to resolve relative paths to actual file locations. """ - errors = super(PathField, self)._validate(*args, ** kwargs) - self.about_file_path = kwargs.get('about_file_path') - self.running_inventory = kwargs.get('running_inventory') - self.base_dir = kwargs.get('base_dir') - self.reference_dir = kwargs.get('reference_dir') + errors = super(PathField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") + self.running_inventory = kwargs.get("running_inventory") + self.base_dir = kwargs.get("base_dir") + self.reference_dir = kwargs.get("reference_dir") if self.base_dir: self.base_dir = util.to_posix(self.base_dir) name = self.name - # Why is the PathField an ordered dict? - # Ans: The reason why the PathField use an ordered dict is because - # for the FileTextField, the key is used as the path to the file and - # the value is used as the context of the file + # Why are paths a dict? + # Ans: The reason why the PathField use a dict is because + # for the FileTextField, the key is used as the path to the file and + # the value is used as the context of the file # dict of normalized paths to a location or None - paths = OrderedDict() + paths = {} for path_value in self.value: - p = path_value.split(',') + p = path_value.split(",") for path in p: path = path.strip() path = util.to_posix(path) @@ -448,23 +493,24 @@ def _validate(self, *args, **kwargs): # normalize eventual / to . # and a succession of one or more ////// to . too if path.strip() and not path.strip(posixpath.sep): - path = '.' - + path = "." + # removing leading and trailing path separator # path are always relative path = path.strip(posixpath.sep) - + # the license files, if need to be copied, are located under the path # set from the 'license-text-location' option, so the tool should check # at the 'license-text-location' instead of the 'base_dir' if not (self.base_dir or self.reference_dir): - msg = (u'Field %(name)s: Unable to verify path: %(path)s:' - u' No base directory provided' % locals()) + msg = ( + "Field %(name)s: Unable to verify path: %(path)s:" + " No base directory provided" % locals() + ) errors.append(Error(ERROR, msg)) location = None paths[path] = location continue - if self.reference_dir: location = posixpath.join(self.reference_dir, path) else: @@ -473,7 +519,7 @@ def _validate(self, *args, **kwargs): if not self.running_inventory and self.about_file_path: # Get the parent directory of the 'about_file_path' afp_parent = posixpath.dirname(self.about_file_path) - + # Create a relative 'about_resource' path by joining the # parent of the 'about_file_path' with the value of the # 'about_resource' @@ -481,25 +527,24 @@ def _validate(self, *args, **kwargs): normalized_arp = posixpath.normpath(arp).strip(posixpath.sep) location = posixpath.join(self.base_dir, normalized_arp) else: - location = posixpath.join(self.base_dir, path) - + location = posixpath.normpath(posixpath.join(self.base_dir, path)) + location = util.to_native(location) location = os.path.abspath(os.path.normpath(location)) location = util.to_posix(location) location = add_unc(location) - + if not os.path.exists(location): # We don't want to show the UNC_PREFIX in the error message location = util.to_posix(location.strip(UNC_PREFIX)) - msg = (u'Field %(name)s: Path %(location)s not found' - % locals()) + msg = "Field %(name)s: Path %(location)s not found" % locals() # We want to show INFO error for 'about_resource' - if name == u'about_resource': + if name == "about_resource": errors.append(Error(INFO, msg)) else: errors.append(Error(CRITICAL, msg)) location = None - + paths[path] = location self.value = paths @@ -511,27 +556,46 @@ class AboutResourceField(PathField): Special field for about_resource. self.resolved_paths contains a list of the paths resolved relative to the about file path. """ - def __init__(self, *args, ** kwargs): - super(AboutResourceField, self).__init__(*args, ** kwargs) + + def __init__(self, *args, **kwargs): + super(AboutResourceField, self).__init__(*args, **kwargs) + self.resolved_paths = [] + + def _validate(self, *args, **kwargs): + errors = super(AboutResourceField, self)._validate(*args, **kwargs) + return errors + + +class IgnoredResourcesField(PathField): + """ + Special field for ignored_resources. self.ignored_paths contains a list of + path patterns (glob patterns) which are not part of the summarization provided + by the ABOUT file. + """ + + def __init__(self, *args, **kwargs): + super(AboutResourceField, self).__init__(*args, **kwargs) self.resolved_paths = [] def _validate(self, *args, **kwargs): - errors = super(AboutResourceField, self)._validate(*args, ** kwargs) + errors = super(AboutResourceField, self)._validate(*args, **kwargs) return errors + class FileTextField(PathField): """ A path field pointing to one or more text files such as license files. The validated value is an ordered dict of path->Text or None if no location or text could not be loaded. """ + def _validate(self, *args, **kwargs): """ Load and validate the texts referenced by paths fields. Return a list of errors. base_dir is the directory used to resolve a file location from a path. """ - errors = super(FileTextField, self)._validate(*args, ** kwargs) + errors = super(FileTextField, self)._validate(*args, **kwargs) # a FileTextField is a PathField # self.value is a paths to location ordered dict # we will replace the location with the text content @@ -545,29 +609,33 @@ def _validate(self, *args, **kwargs): try: # TODO: we have lots the location by replacing it with a text location = add_unc(location) - with io.open(location, encoding='utf-8') as txt: + with open(location, encoding="utf-8", errors="replace") as txt: text = txt.read() self.value[path] = text except Exception as e: # only keep the first 100 char of the exception emsg = repr(e)[:100] - msg = (u'Field %(name)s: Failed to load text at path: ' - u'%(path)s ' - u'with error: %(emsg)s' % locals()) + msg = ( + "Field %(name)s: Failed to load text at path: " + "%(path)s " + "with error: %(emsg)s" % locals() + ) errors.append(Error(ERROR, msg)) # set or reset self self.errors = errors return errors + class BooleanField(SingleLineField): """ An flag field with a boolean value. Validated value is False, True or None. """ + def default_value(self): return None - true_flags = ('yes', 'y', 'true', 'x') - false_flags = ('no', 'n', 'false') + true_flags = ("yes", "y", "true", "x") + false_flags = ("no", "n", "false") flag_values = true_flags + false_flags def _validate(self, *args, **kwargs): @@ -575,25 +643,27 @@ def _validate(self, *args, **kwargs): Check that flag are valid. Convert flags to booleans. Default flag to False. Return a list of errors. """ - errors = super(BooleanField, self)._validate(*args, ** kwargs) - self.about_file_path = kwargs.get('about_file_path') + errors = super(BooleanField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") flag = self.get_flag(self.original_value) if flag is False: name = self.name val = self.original_value about_file_path = self.about_file_path flag_values = self.flag_values - msg = (u'Path: %(about_file_path)s - Field %(name)s: Invalid flag value: %(val)r is not ' - u'one of: %(flag_values)s' % locals()) + msg = ( + "Path: %(about_file_path)s - Field %(name)s: Invalid flag value: %(val)r is not " + "one of: %(flag_values)s" % locals() + ) errors.append(Error(ERROR, msg)) self.value = None elif flag is None: name = self.name - msg = (u'Field %(name)s: field is present but empty. ' % locals()) + msg = "Field %(name)s: field is present but empty. " % locals() errors.append(Error(INFO, msg)) self.value = None else: - if flag == u'yes' or flag is True: + if flag == "yes" or flag is True: self.value = True else: self.value = False @@ -605,13 +675,13 @@ def get_flag(self, value): possible values or None if empty or False if not found or original value if it is not a boolean value """ - if value is None or value == '': + if value is None or value == "": return None if isinstance(value, bool): return value else: - if isinstance(value, basestring): + if isinstance(value, str): value = value.strip() if not value: return None @@ -619,9 +689,9 @@ def get_flag(self, value): value = value.lower() if value in self.flag_values: if value in self.true_flags: - return u'yes' + return "yes" else: - return u'no' + return "no" else: return False else: @@ -639,25 +709,126 @@ def has_content(self): def _serialized_value(self): # default normalized values for serialization if self.value: - return u'yes' + return "yes" elif self.value is False: - return u'no' + return "no" else: # self.value is None # TODO: should we serialize to No for None??? - return u'' + return "" def __eq__(self, other): """ Boolean equality """ - return (isinstance(other, self.__class__) - and self.name == other.name - and self.value == other.value) + return ( + isinstance(other, self.__class__) + and self.name == other.name + and self.value == other.value + ) -def validate_fields(fields, about_file_path, running_inventory, base_dir, - reference_dir=None): +class BooleanAndTwoCharactersField(SingleLineField): + """ + Field with either a boolean value or character(s) value (at most 2 + characters). Validated value is False, True, None or character value. + """ + + def default_value(self): + return None + + true_flags = ("yes", "y", "true", "x") + false_flags = ("no", "n", "false") + flag_values = true_flags + false_flags + + def _validate(self, *args, **kwargs): + """ + Check that flag are valid with either boolean value or character value. + Default flag to False. Return a list of errors. + """ + errors = super(BooleanAndTwoCharactersField, self)._validate(*args, **kwargs) + self.about_file_path = kwargs.get("about_file_path") + flag = self.get_value(self.original_value) + if flag is False: + name = self.name + val = self.original_value + about_file_path = self.about_file_path + flag_values = self.flag_values + msg = ( + "Path: %(about_file_path)s - Field %(name)s: Invalid value: %(val)r is not " + "one of: %(flag_values)s and it is not a 1 or 2 character value." % locals() + ) + errors.append(Error(ERROR, msg)) + self.value = None + elif flag is None: + name = self.name + msg = "Field %(name)s: field is present but empty. " % locals() + errors.append(Error(INFO, msg)) + self.value = None + else: + if flag == "yes" or flag is True: + self.value = True + elif flag == "no": + self.value = False + else: + self.value = flag + return errors + + def get_value(self, value): + """ + Return a normalized existing value if found in the list of + possible values or None if empty or False if not found or original value + if it is not a boolean value + """ + if value is None or value == "": + return None + + if isinstance(value, bool): + return value + else: + if isinstance(value, str): + value = value.strip() + if not value: + return None + + value = value.lower() + if value in self.flag_values or len(value) <= 2: + if value in self.true_flags: + return "yes" + elif value in self.false_flags: + return "no" + else: + return value + else: + return False + else: + return False + + @property + def has_content(self): + """ + Return true if it has content regardless of what value, False otherwise + """ + if self.original_value: + return True + return False + + def _serialized_value(self): + # default normalized values for serialization + if self.value: + if isinstance(self.value, bool): + return "yes" + else: + return self.value + elif self.value is False: + return "no" + else: + # self.value is None + # TODO: should we serialize to No for None??? + return "" + + +def validate_fields(fields, about_file_path, running_inventory, base_dir, reference_dir=None): """ Validate a sequence of Field objects. Return a list of errors. Validation may update the Field objects as needed as a side effect. @@ -676,29 +847,45 @@ def validate_fields(fields, about_file_path, running_inventory, base_dir, def validate_field_name(name): if not is_valid_name(name): - msg = ('Field name: %(name)r contains illegal name characters: ' - '0 to 9, a to z, A to Z and _.') - return Error(CRITICAL, msg % locals()) + msg = ( + "Field name: %(name)r contains illegal name characters " + "(or empty spaces) and is ignored." + ) + return Error(WARNING, msg % locals()) + + +class License: + """ + Represent a License object + """ + + def __init__(self, key, name, filename, url, text): + self.key = key + self.name = name + self.filename = filename + self.url = url + self.text = text class About(object): """ Represent an ABOUT file and functions to parse and validate a file. """ + # special names, used only when serializing lists of ABOUT files to CSV or # similar # name of the attribute containing the relative ABOUT file path - ABOUT_FILE_PATH_ATTR = 'about_file_path' + ABOUT_FILE_PATH_ATTR = "about_file_path" # name of the attribute containing the resolved relative Resources paths - about_resource_path_attr = 'about_resource_path' - + about_resource_path_attr = "about_resource_path" + # name of the attribute containing the resolved relative Resources paths - ABOUT_RESOURCE_ATTR = 'about_resource' + ABOUT_RESOURCE_ATTR = "about_resource" # Required fields - required_fields = ['name', ABOUT_RESOURCE_ATTR] + required_fields = ["name"] def get_required_fields(self): return [f for f in self.fields if f.required] @@ -709,51 +896,52 @@ def set_standard_fields(self): could use a metaclass to track ordering django-like but this approach is simpler. """ - self.fields = OrderedDict([ - ('about_resource', AboutResourceField(required=True)), - ('name', SingleLineField(required=True)), - ('version', SingleLineField()), - - ('download_url', UrlField()), - ('description', StringField()), - ('homepage_url', UrlField()), - ('notes', StringField()), - - ('license_expression', StringField()), - ('license_key', ListField()), - ('license_name', ListField()), - ('license_file', FileTextField()), - ('license_url', UrlListField()), - ('copyright', StringField()), - ('notice_file', FileTextField()), - ('notice_url', UrlField()), - - ('redistribute', BooleanField()), - ('attribute', BooleanField()), - ('track_changes', BooleanField()), - ('modified', BooleanField()), - ('internal_use_only', BooleanField()), - - ('changelog_file', FileTextField()), - - ('owner', StringField()), - ('owner_url', UrlField()), - ('contact', StringField()), - ('author', StringField()), - ('author_file', FileTextField()), - - ('vcs_tool', SingleLineField()), - ('vcs_repository', SingleLineField()), - ('vcs_path', SingleLineField()), - ('vcs_tag', SingleLineField()), - ('vcs_branch', SingleLineField()), - ('vcs_revision', SingleLineField()), - - ('checksum_md5', SingleLineField()), - ('checksum_sha1', SingleLineField()), - ('checksum_sha256', SingleLineField()), - ('spec_version', SingleLineField()), - ]) + self.fields = dict( + [ + ("about_resource", AboutResourceField()), + ("ignored_resources", AboutResourceField()), + ("name", SingleLineField(required=True)), + ("version", SingleLineField()), + ("download_url", UrlField()), + ("description", StringField()), + ("homepage_url", UrlField()), + ("package_url", PackageUrlField()), + ("notes", StringField()), + ("license_expression", SingleLineField()), + ("license_key", ListField()), + ("license_name", ListField()), + ("license_file", FileTextField()), + ("license_url", UrlListField()), + ("spdx_license_expression", SingleLineField()), + ("spdx_license_key", ListField()), + ("declared_license_expression", SingleLineField()), + ("other_license_expression", SingleLineField()), + ("copyright", StringField()), + ("notice_file", FileTextField()), + ("notice_url", UrlField()), + ("redistribute", BooleanField()), + ("attribute", BooleanAndTwoCharactersField()), + ("track_changes", BooleanField()), + ("modified", BooleanField()), + ("internal_use_only", BooleanField()), + ("changelog_file", FileTextField()), + ("owner", StringField()), + ("owner_url", UrlField()), + ("contact", StringField()), + ("author", StringField()), + ("author_file", FileTextField()), + ("vcs_tool", SingleLineField()), + ("vcs_repository", SingleLineField()), + ("vcs_path", SingleLineField()), + ("vcs_tag", SingleLineField()), + ("vcs_branch", SingleLineField()), + ("vcs_revision", SingleLineField()), + ("checksum_md5", SingleLineField()), + ("checksum_sha1", SingleLineField()), + ("checksum_sha256", SingleLineField()), + ("spec_version", SingleLineField()), + ] + ) for name, field in self.fields.items(): # we could have a hack to get the actual field name @@ -768,7 +956,7 @@ def __init__(self, location=None, about_file_path=None, strict=False): attribute contains the errors. """ self.set_standard_fields() - self.custom_fields = OrderedDict() + self.custom_fields = {} self.errors = [] @@ -783,7 +971,7 @@ def __init__(self, location=None, about_file_path=None, strict=False): self.base_dir = os.path.dirname(location) self.errors.extend(self.load(location)) if strict and self.errors and filter_errors(self.errors): - msg = '\n'.join(map(str, self.errors)) + msg = "\n".join(map(str, self.errors)) raise Exception(msg) def __repr__(self): @@ -793,9 +981,11 @@ def __eq__(self, other): """ Equality based on fields and custom_fields., i.e. content. """ - return (isinstance(other, self.__class__) - and self.fields == other.fields - and self.custom_fields == other.custom_fields) + return ( + isinstance(other, self.__class__) + and self.fields == other.fields + and self.custom_fields == other.custom_fields + ) def all_fields(self): """ @@ -808,7 +998,7 @@ def as_dict(self): Return all the standard fields and customer-defined fields of this About object in an ordered dict. """ - data = OrderedDict() + data = {} data[self.ABOUT_FILE_PATH_ATTR] = self.about_file_path with_values = ((fld.name, fld.serialized_value()) for fld in self.all_fields()) non_empty = ((name, value) for name, value in with_values if value) @@ -822,7 +1012,8 @@ def hydrate(self, fields): Return a list of errors. """ errors = [] - seen_fields = OrderedDict() + seen_fields = {} + illegal_name_list = [] for name, value in fields: orig_name = name @@ -838,13 +1029,15 @@ def hydrate(self, fields): # this is a special attribute, skip entirely continue - # A field that has been alredy processed ... and has a value + # A field that has been already processed ... and has a value previous_value = seen_fields.get(name) if previous_value: if value != previous_value: - msg = (u'Field %(orig_name)s is a duplicate. ' - u'Original value: "%(previous_value)s" ' - u'replaced with: "%(value)s"') + msg = ( + "Field %(orig_name)s is a duplicate. " + 'Original value: "%(previous_value)s" ' + 'replaced with: "%(value)s"' + ) errors.append(Error(WARNING, msg % locals())) continue @@ -860,12 +1053,12 @@ def hydrate(self, fields): # A custom field # is the name valid? - illegal_name_error = validate_field_name(name) - if illegal_name_error: - errors.append(illegal_name_error) + if not is_valid_name(name): + if not name in illegal_name_list: + illegal_name_list.append(name) continue - msg = 'Field %(orig_name)s is a custom field.' + msg = "Custom Field: %(orig_name)s" errors.append(Error(INFO, msg % locals())) # is this a known one? custom_field = self.custom_fields.get(name) @@ -876,23 +1069,35 @@ def hydrate(self, fields): custom_field.present = True else: # A new, unknown custom field - # custom fields are always handled as StringFields - # FIXME: with yaml we could just set whatever is provided - custom_field = StringField(name=name, value=value, present=True) + custom_field = Field(name=name, value=value, present=True) self.custom_fields[name] = custom_field # FIXME: why would this ever fail??? try: if name in dir(self): - raise Exception('Illegal field: %(name)r: %(value)r.' % locals()) + raise Exception("Illegal field: %(name)r: %(value)r." % locals()) setattr(self, name, custom_field) except: - msg = 'Internal error with custom field: %(name)r: %(value)r.' + msg = "Internal error with custom field: %(name)r: %(value)r." errors.append(Error(CRITICAL, msg % locals())) + if illegal_name_list: + msg = ( + "Field name: %(illegal_name_list)r contains illegal name characters " + "(or empty spaces) and is ignored." + ) + errors.append(Error(WARNING, msg % locals())) return errors - def process(self, fields, about_file_path, running_inventory=False, - base_dir=None, reference_dir=None): + def process( + self, + fields, + about_file_path, + running_inventory=False, + base_dir=None, + scancode=False, + from_attrib=False, + reference_dir=None, + ): """ Validate and set as attributes on this About object a sequence of `fields` name/value tuples. Return a list of errors. @@ -902,19 +1107,26 @@ def process(self, fields, about_file_path, running_inventory=False, afp = self.about_file_path errors = self.hydrate(fields) + # We want to copy the license_files before the validation - if reference_dir: - copy_license_notice_files( - fields, base_dir, reference_dir, afp) + if reference_dir and not from_attrib: + copy_err = copy_license_notice_files(fields, base_dir, reference_dir, afp) + errors.extend(copy_err) # TODO: why? we validate all fields, not only these hydrated - validation_errors = validate_fields( - self.all_fields(), - about_file_path, - running_inventory, - self.base_dir, - self.reference_dir) - errors.extend(validation_errors) + # The validate functions does not allow duplicated entry for a list meaning + # it will cause problem when using scancode license detection as an input as + # it usually returns duplicated license_key and many license have duplicated + # score such as 100. We need to handle this scenario using different method. + if not scancode: + validation_errors = validate_fields( + self.all_fields(), + about_file_path, + running_inventory, + self.base_dir, + self.reference_dir, + ) + errors.extend(validation_errors) return errors def load(self, location): @@ -928,8 +1140,13 @@ def load(self, location): errors = [] try: loc = add_unc(loc) - with io.open(loc, encoding='utf-8') as txt: + with open(loc, encoding="utf-8", errors="replace") as txt: input_text = txt.read() + if not input_text: + msg = "ABOUT file is empty: %(location)r" + errors.append(Error(CRITICAL, msg % locals())) + self.errors = errors + return errors # The 'Yes' and 'No' will be converted to 'True' and 'False' in the yaml.load() # Therefore, we need to wrap the original value in quote to prevent # the conversion @@ -950,11 +1167,14 @@ def load(self, location): """ running_inventory = True data = saneyaml.load(input, allow_duplicate_keys=False) - errs = self.load_dict(data, base_dir, running_inventory) + errs = self.load_dict(data, base_dir, running_inventory=running_inventory) errors.extend(errs) except Exception as e: - trace = traceback.format_exc() - msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s' + # The trace is good for debugging, but probably not good for user to + # see the traceback message + # trace = traceback.format_exc() + # msg = 'Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s' + msg = "Cannot load invalid ABOUT file: %(location)r: %(e)r" errors.append(Error(CRITICAL, msg % locals())) self.errors = errors @@ -962,7 +1182,16 @@ def load(self, location): # FIXME: should be a from_dict class factory instead # FIXME: running_inventory: remove this : this should be done in the commands, not here - def load_dict(self, fields_dict, base_dir, running_inventory=False, reference_dir=None,): + + def load_dict( + self, + fields_dict, + base_dir, + scancode=False, + from_attrib=False, + running_inventory=False, + reference_dir=None, + ): """ Load this About object file from a `fields_dict` name/value dict. Return a list of errors. @@ -970,39 +1199,99 @@ def load_dict(self, fields_dict, base_dir, running_inventory=False, reference_di # do not keep empty fields = list(fields_dict.items()) - for key, value in fields: - if not value: - # never return empty or absent fieds - continue - if key == u'licenses': - # FIXME: use a license object instead - lic_key, lic_name, lic_file, lic_url = ungroup_licenses(value) - if lic_key: - fields.append(('license_key', lic_key)) - if lic_name: - fields.append(('license_name', lic_name)) - if lic_file: - fields.append(('license_file', lic_file)) - if lic_url: - fields.append(('license_url', lic_url)) - # The licenses field has been ungrouped and can be removed. - # Otherwise, it will gives the following INFO level error - # 'Field licenses is a custom field.' - licenses_field = (key, value) - fields.remove(licenses_field) + if scancode: + have_copyright = False + for key, value in fields: + if not value: + continue + if key == "copyrights": + have_copyright = True + elif key == "license_detections": + lic_list = ungroup_licenses_from_sctk(value) + lic_exp_list = [] + for detected_license in value: + if "license_expression" in detected_license: + lic_exp_list.append(detected_license["license_expression"]) + if lic_exp_list: + fields.append(("license_expression", " AND ".join(lic_exp_list))) + + lic_key_list = [] + lic_key_exp_list = [] + lic_score_list = [] + for lic in lic_list: + _char, lic_keys, _invalid_lic_exp = parse_license_expression(lic["lic_exp"]) + lic_key_list.append(lic_keys) + # for lic_key in lic_keys: + # lic_key_list.append([lic_key]) + + for lic in lic_list: + lic_key_exp_list.append(lic["lic_exp"]) + lic_score_list.append(lic["score"]) + fields.append(("license_key", lic_key_list)) + fields.append(("license_key_expression", lic_key_exp_list)) + fields.append(("license_score", lic_score_list)) + + # The licenses field has been ungrouped and can be removed. + # Otherwise, it will gives the following INFO level error + # 'Field licenses is a custom field.' + licenses_field = (key, value) + fields.remove(licenses_field) + # Make sure the copyrights is present even is empty to avoid error + # when generating with Jinja + if not have_copyright: + fields.append(("copyrights", "")) + + else: + for key, value in fields: + if not value: + # never return empty or absent fields + continue + if key == "licenses": + # FIXME: use a license object instead + ( + lic_key, + lic_name, + lic_file, + lic_url, + spdx_lic_key, + lic_score, + lic_matched_text, + ) = ungroup_licenses(value) + if lic_key: + fields.append(("license_key", lic_key)) + if lic_name: + fields.append(("license_name", lic_name)) + if lic_file: + fields.append(("license_file", lic_file)) + if lic_url: + fields.append(("license_url", lic_url)) + if spdx_lic_key: + fields.append(("spdx_license_key", spdx_lic_key)) + # The license score is a key from scancode license scan + if lic_score: + fields.append(("license_score", lic_score)) + if lic_matched_text: + fields.append(("matched_text", lic_matched_text)) + # The licenses field has been ungrouped and can be removed. + # Otherwise, it will gives the following INFO level error + # 'Field licenses is a custom field.' + licenses_field = (key, value) + fields.remove(licenses_field) errors = self.process( fields=fields, about_file_path=self.about_file_path, running_inventory=running_inventory, base_dir=base_dir, + scancode=scancode, + from_attrib=from_attrib, reference_dir=reference_dir, ) self.errors = errors return errors @classmethod - def from_dict(cls, about_data, base_dir=''): + def from_dict(cls, about_data, base_dir=""): """ Return an About object loaded from a python dict. """ @@ -1010,38 +1299,59 @@ def from_dict(cls, about_data, base_dir=''): about.load_dict(about_data, base_dir=base_dir) return about - def dumps(self): + def dumps(self, licenses_dict=None): """ Return self as a formatted ABOUT string. """ - data = OrderedDict() + data = {} # Group the same license information (name, url, file) together license_key = [] license_name = [] license_file = [] license_url = [] - bool_fields = ['redistribute', 'attribute', 'track_changes', 'modified', 'internal_use_only'] + spdx_license_key = [] + bool_fields = [ + "redistribute", + "attribute", + "track_changes", + "modified", + "internal_use_only", + ] for field in self.all_fields(): if not field.value and not field.name in bool_fields: continue - - if field.name == 'license_key' and field.value: + if field.name == "license_key" and field.value: license_key = field.value - elif field.name == 'license_name' and field.value: + elif field.name == "license_name" and field.value: license_name = field.value - elif field.name == 'license_file' and field.value: + elif field.name == "license_file" and field.value: # Restore the original_value as it was parsed for # validation purpose if field.original_value: - license_file = field.original_value.split('\n') + # This line break is for the components that have multiple license + # values in CSV format. + if "\n" in field.original_value: + license_file_list = field.original_value.split("\n") + license_file = [] + # Strip the carriage return character '\r' See #443 + for lic in license_file_list: + if "\r" in lic: + license_file.append(lic.strip("\r")) + else: + license_file.append(lic) + else: + if isinstance(field.original_value, list): + license_file = list(field.value.keys()) + else: + # Restore the original license_file value + # See #444 + license_file = [field.original_value] else: - license_file = field.value.keys() - elif field.name == 'license_url' and field.value: + license_file = list(field.value.keys()) + elif field.name == "license_url" and field.value: license_url = field.value - - # No multiple 'about_resource' reference supported. - # Take the first element (should only be one) in the list for the - # value of 'about_resource' + elif field.name == "spdx_license_key" and field.value: + spdx_license_key = field.value elif field.name in file_fields and field.value: data[field.name] = field.original_value elif field.name in bool_fields and not field.value == None: @@ -1049,24 +1359,99 @@ def dumps(self): else: if field.value: data[field.name] = field.value + # If there is no license_key value, parse the license_expression + # and get the parsed license key + if "license_expression" in data: + if not license_key and data["license_expression"]: + _spec_char, lic_list, _invalid_lic_exp = parse_license_expression( + data["license_expression"] + ) + license_key = lic_list # Group the same license information in a list - license_group = list(zip_longest(license_key, license_name, license_file, license_url)) + # This `licenses_dict` is a dictionary with license key as the key and the + # value is the list of [license_name, license_filename, license_context, license_url] + lic_key_copy = license_key[:] + lic_dict_list = [] + for lic_key in license_key: + lic_dict = {} + if licenses_dict and lic_key in licenses_dict: + lic_dict["key"] = lic_key + lic_name, lic_filename, lic_context, lic_url, spdx_lic_key = licenses_dict[lic_key] + if lic_name: + lic_dict["name"] = lic_name + if lic_filename: + lic_dict["file"] = lic_filename + if lic_url: + lic_dict["url"] = lic_url + if spdx_lic_key: + lic_dict["spdx_license_key"] = spdx_lic_key + + # Remove the license information if it has been handled + # The following condition is to check if license information + # has been fetched, the license key is invalid or custom if + # no value for lic_name + if lic_name: + lic_key_copy.remove(lic_key) + if lic_name in license_name: + license_name.remove(lic_name) + if lic_url in license_url: + license_url.remove(lic_url) + if lic_filename in license_file: + license_file.remove(lic_filename) + if spdx_lic_key in spdx_license_key: + spdx_license_key.remove(spdx_lic_key) + lic_dict_list.append(lic_dict) + + # Handle license information that have not been handled. + # If the len of the lic_key is the same as the lic_file, the tool should + # assume the lic_file (custom license) is referring this specific lic_key + # otherwise, the tool shouldn't group them + if len(lic_key_copy) == len(license_file): + license_group = list( + zip_longest(lic_key_copy, license_name, license_file, license_url, spdx_license_key) + ) + else: + license_group = list( + zip_longest(lic_key_copy, license_name, [], license_url, spdx_license_key) + ) + # Add the unhandled_lic_file if any + if license_file: + for lic_file in license_file: + license_group.append((None, None, lic_file, None, None)) + for lic_group in license_group: - lic_dict = OrderedDict() + lic_dict = {} if lic_group[0]: - lic_dict['key'] = lic_group[0] + lic_dict["key"] = lic_group[0] if lic_group[1]: - lic_dict['name'] = lic_group[1] + lic_dict["name"] = lic_group[1] + else: + # If no name is given, treat the key as the name + if lic_group[0]: + lic_dict["name"] = lic_group[0] if lic_group[2]: - lic_dict['file'] = lic_group[2] + lic_dict["file"] = lic_group[2] if lic_group[3]: - lic_dict['url'] = lic_group[3] - data.setdefault('licenses', []).append(lic_dict) + lic_dict["url"] = lic_group[3] + if lic_group[4]: + lic_dict["spdx_license_key"] = lic_group[4] + lic_dict_list.append(lic_dict) + + # Format the license information in the same order of the license expression + for key in license_key: + for lic_dict in lic_dict_list: + if key == lic_dict["key"]: + data.setdefault("licenses", []).append(lic_dict) + lic_dict_list.remove(lic_dict) + break + + for lic_dict in lic_dict_list: + data.setdefault("licenses", []).append(lic_dict) return saneyaml.dump(data) - def dump(self, location): + def dump(self, location, lic_dict=None): """ Write formatted ABOUT representation of self to location. """ @@ -1077,26 +1462,75 @@ def dump(self, location): os.makedirs(add_unc(parent)) about_file_path = loc - if not about_file_path.endswith('.ABOUT'): + if not about_file_path.endswith(".ABOUT"): # FIXME: we should not infer some location. - if about_file_path.endswith('/'): - about_file_path = util.to_posix( - os.path.join(parent, os.path.basename(parent))) - about_file_path += '.ABOUT' + if about_file_path.endswith("/"): + about_file_path = util.to_posix(os.path.join(parent, os.path.basename(parent))) + about_file_path += ".ABOUT" if on_windows: about_file_path = add_unc(about_file_path) - with io.open(about_file_path, mode='w', encoding='utf-8') as dumped: + with open(about_file_path, mode="w", encoding="utf-8", errors="replace") as dumped: dumped.write(genereated_tk_version) - dumped.write(self.dumps()) + dumped.write(self.dumps(lic_dict)) + + def dump_android_notice(self, path, context): + """ + Write the NOITCE file consist of copyright, notice and license + """ + if on_windows: + path = add_unc(path) + + with open(path, mode="w", encoding="utf-8", errors="replace") as dumped: + dumped.write(context) + + def android_module_license(self, about_parent_path): + """ + Create MODULE_LICENSE_XXX which the XXX is the value of license key. + """ + for lic_key in self.license_key.value: + # Make uppercase and with dash and spaces and dots replaced by underscore + # just to look similar and consistent. + name = ( + "MODULE_LICENSE_" + + lic_key.replace(".", "_").replace("-", "_").replace(" ", "_").upper() + ) + module_lic_path = os.path.join(about_parent_path, name) + # Create an empty MODULE_LICESE_XXX file + open(module_lic_path, "a").close() + + def android_notice(self, about_parent_path): + """ + Return a notice dictionary which the path of the notice file going + to create will be the key and its context will be the value of the dict. + """ + # Create NOTICE file with the combination context of copyright, + # notice_file and license_file + notice_path = posixpath.join(about_parent_path, "NOTICE") + notice_context = "" + if self.copyright.value: + notice_context += self.copyright.value + if self.notice_file.value: + notice_file_dict = self.notice_file.value + notice_file_key = notice_file_dict.keys() + for key in notice_file_key: + if notice_file_dict[key]: + notice_context += "\n" + notice_file_dict[key] + "\n" + if self.license_file.value: + lic_file_dict = self.license_file.value + lic_file_key = lic_file_dict.keys() + for key in lic_file_key: + if lic_file_dict[key]: + notice_context += "\n\n" + lic_file_dict[key] + "\n\n" + return notice_path, notice_context def dump_lic(self, location, license_dict): """ Write LICENSE files and return the a list of key, name, context and the url as these information are needed for the ABOUT file """ - license_name = license_context = license_url = '' + license_name = license_context = license_url = "" loc = util.to_posix(location) parent = posixpath.dirname(loc) license_key_name_context_url = [] @@ -1104,48 +1538,173 @@ def dump_lic(self, location, license_dict): if not posixpath.exists(parent): os.makedirs(add_unc(parent)) - if self.license_expression.present and not self.license_file.present: - special_char_in_expression, lic_list = parse_license_expression(self.license_expression.value) - self.license_key.value = lic_list + licenses_list = [] + if self.license_expression.present: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + self.license_expression.value + ) + if lic_list: + for lic in lic_list: + if lic not in licenses_list: + licenses_list.append(lic) + if self.declared_license_expression.present: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + self.declared_license_expression.value + ) + if lic_list: + for lic in lic_list: + if lic not in licenses_list: + licenses_list.append(lic) + if self.other_license_expression.present: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + self.other_license_expression.value + ) + if lic_list: + for lic in lic_list: + if lic not in licenses_list: + licenses_list.append(lic) + if licenses_list: + self.license_key.value = licenses_list self.license_key.present = True - if not special_char_in_expression: - for lic_key in lic_list: - try: - if license_dict[lic_key]: - license_path = posixpath.join(parent, lic_key) - license_path += u'.LICENSE' - license_path = add_unc(license_path) - license_name, license_context, license_url = license_dict[lic_key] - license_info = (lic_key, license_name, license_context, license_url) - license_key_name_context_url.append(license_info) - with io.open(license_path, mode='w', encoding='utf-8', newline='\n') as lic: - lic.write(license_context) - except: - pass + for lic_key in licenses_list: + license_name = "" + license_filename = "" + license_context = "" + license_url = "" + spdx_license_key = "" + if lic_key in license_dict: + license_path = posixpath.join(parent, lic_key) + license_path += ".LICENSE" + license_path = add_unc(license_path) + ( + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) = license_dict[lic_key] + license_info = ( + lic_key, + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) + license_key_name_context_url.append(license_info) + with open( + license_path, mode="w", encoding="utf-8", newline="\n", errors="replace" + ) as lic: + lic.write(license_context) + else: + # Invalid license issue is already handled + license_info = ( + lic_key, + license_name, + license_filename, + license_context, + license_url, + spdx_license_key, + ) + license_key_name_context_url.append(license_info) + return license_key_name_context_url -def collect_inventory(location): +def collect_inventory(location, exclude=None): """ Collect ABOUT files at location and return a list of errors and a list of About objects. """ errors = [] input_location = util.get_absolute(location) - about_locations = list(util.get_about_locations(input_location)) + about_locations = list(util.get_about_locations(input_location, exclude)) name_errors = util.check_file_names(about_locations) errors.extend(name_errors) abouts = [] + custom_fields_list = [] for about_loc in about_locations: about_file_path = util.get_relative_path(input_location, about_loc) about = About(about_loc, about_file_path) - # Insert about_file_path reference to the error for severity, message in about.errors: - msg = (about_file_path + ": " + message) - errors.append(Error(severity, msg)) + if "Custom Field" in message: + field_name = message.replace("Custom Field: ", "").strip() + if field_name not in custom_fields_list: + custom_fields_list.append(field_name) + else: + msg = about_file_path + ": " + message + errors.append(Error(severity, msg)) abouts.append(about) - return unique(errors), abouts + if custom_fields_list: + custom_fields_err_msg = "Field " + str(custom_fields_list) + " is a custom field." + errors.append(Error(INFO, custom_fields_err_msg)) + return errors, abouts + + +def collect_abouts_license_expression(location): + """ + Read the ABOUT files at location and return a list of ABOUT objects without + validation. The purpose of this is to speed up the process for `gen_license` command. + """ + lic_key_list = [] + errors = [] + input_location = util.get_absolute(location) + about_locations = list(util.get_about_locations(input_location)) + abouts = [] + + for loc in about_locations: + try: + loc = add_unc(loc) + with open(loc, encoding="utf-8", errors="replace") as txt: + input_text = txt.read() + # saneyaml.load() will have parsing error if the input has + # tab value. Therefore, we should check if the input contains + # any tab and then convert it to spaces. + input = replace_tab_with_spaces(input_text) + data = saneyaml.load(input, allow_duplicate_keys=False) + about = About() + about.load_dict(data, base_dir="") + abouts.append(about) + except Exception as e: + trace = traceback.format_exc() + msg = "Cannot load invalid ABOUT file: %(location)r: %(e)r\n%(trace)s" + errors.append(Error(CRITICAL, msg % locals())) + + return errors, abouts + + +def collect_inventory_license_expression(location, scancode=False, worksheet=None): + """ + Read the inventory file at location and return a list of ABOUT objects without + validation. The purpose of this is to speed up the process for `gen_license` command. + """ + abouts = [] + errors = [] + + if scancode: + inventory = gen.load_scancode_json(location) + # ScanCode uses 'detected_license_expression' + if not "detected_license_expression" in inventory[0]: + errors.append(Error(CRITICAL, "No 'license_expressions' field in the input.")) + return errors, abouts + else: + if location.endswith(".csv"): + inventory = gen.load_csv(location) + elif location.endswith(".xlsx"): + _dup_cols_err, inventory = gen.load_excel(location, worksheet) + else: + inventory = gen.load_json(location) + # Check if 'license_expression' field is in the input + if not inventory or not "license_expression" in inventory[0]: + errors.append(Error(CRITICAL, "No 'license_expression' field in the input.")) + return errors, abouts + + for data in inventory: + about = About() + about.load_dict(data, base_dir="", scancode=scancode) + abouts.append(about) + return errors, abouts def get_field_names(abouts): @@ -1188,6 +1747,114 @@ def get_field_names(abouts): return fields +def copy_redist_src(copy_list, location, output, with_structure): + """ + Given a list of files/directories and copy to the destination + """ + errors = [] + for from_path in copy_list: + norm_from_path = norm(from_path) + relative_from_path = norm_from_path.partition(util.norm(location))[2] + # Need to strip the '/' to use the join + if relative_from_path.startswith("/"): + relative_from_path = relative_from_path.partition("/")[2] + # Get the directory name of the output path + if with_structure: + output_dir = os.path.dirname(os.path.join(output, util.norm(relative_from_path))) + else: + output_dir = output + err = copy_file(from_path, output_dir) + if err: + errors.extend(err) + return errors + + +def get_copy_list(abouts, location): + """ + Return a list of files/directories that need to be copied (and error if any) + This is a summary list in a sense that if a directory is already in the list, + its children directories/files will not be included in the list regardless if + they have 'redistribute' flagged. The reason for this is we want to capture + the error/warning if existence files/directories already exist. However, if + we don't have this "summarized" list, and we've copied a file (with directory structure) + and then later on this file's parent directory also need to be copied, then + it will prompt warning as the directory that need to be copied is already exist. + Technically, this is correct, but it leads to confusion. Therefore, we want to + create a summarized list to avoid this kind of confusion. + """ + errors = [] + copy_list = [] + dir_list = [] + file_list = [] + for about in abouts: + if about.redistribute.value: + file_exist = True + for e in about.errors: + if "Field about_resource" in e.message and "not found" in e.message: + msg = e.message + " and cannot be copied." + errors.append(Error(CRITICAL, msg)) + file_exist = False + continue + if file_exist: + for k in about.about_resource.value: + from_path = about.about_resource.value.get(k) + if on_windows: + norm_from_path = norm(from_path) + else: + norm_from_path = os.path.normpath(from_path) + # Get the relative path + relative_from_path = norm_from_path.partition(util.norm(location))[2] + if os.path.isdir(from_path): + if not dir_list: + dir_list.append(relative_from_path) + else: + handled = False + for dir in dir_list: + # The dir is a parent of the relative_from_path + if dir in relative_from_path: + handled = True + continue + # The relative_from_path is the parent of the dir + # We need to update the dir_list + if relative_from_path in dir: + dir_list.remove(dir) + dir_list.append(relative_from_path) + handled = True + continue + if not handled: + dir_list.append(relative_from_path) + else: + # Check if the file is from "root" + # If the file is at root level, it'll add to the copy_list + if not os.path.dirname(relative_from_path) == "/": + file_list.append(relative_from_path) + else: + copy_list.append(from_path) + + for dir in dir_list: + for f in file_list: + # The file is already in one of copied directories + if dir in f: + file_list.remove(f) + continue + if dir.startswith("/"): + dir = dir.partition("/")[2] + absolute_path = os.path.join(location, dir) + if on_windows: + absolute_path = add_unc(absolute_path) + copy_list.append(absolute_path) + + for f in file_list: + if f.startswith("/"): + f = f.partition("/")[2] + absolute_path = os.path.join(location, f) + if on_windows: + absolute_path = add_unc(absolute_path) + copy_list.append(absolute_path) + + return copy_list, errors + + def about_object_to_list_of_dictionary(abouts): """ Convert About objects to a list of dictionaries @@ -1196,36 +1863,34 @@ def about_object_to_list_of_dictionary(abouts): for about in abouts: # Restore the *_file value to the original value # The *_file's original_value may be parsed (i.e. split(',)) - # for validation purpose. + # for validation purpose. about.license_file.value = about.license_file.original_value - about.notice_file.value = about.notice_file.original_value + about.notice_file.value = about.notice_file.original_value about.changelog_file.value = about.changelog_file.original_value about.author_file.value = about.author_file.original_value # TODO: this wholeblock should be under sd_dict() ad = about.as_dict() - # Update the 'about_resource' field with the relative path - # from the output location - try: - if ad['about_resource']: - if 'about_file_path' in ad.keys(): - afp = ad['about_file_path'] - afp_parent = posixpath.dirname(afp) - afp_parent = '/' + afp_parent if not afp_parent.startswith('/') else afp_parent - about_resource = ad['about_resource'] - for resource in about_resource: - updated_about_resource = posixpath.normpath(posixpath.join(afp_parent, resource)) - if resource == u'.': - if not updated_about_resource == '/': - updated_about_resource = updated_about_resource + '/' - ad['about_resource'] = OrderedDict([(updated_about_resource, None)]) - del ad['about_file_path'] - serialized.append(ad) - except Exception as e: - # The missing required field, about_resource, has already been checked - # and the error has already been logged. - pass + if "about_file_path" in ad.keys(): + afp = ad["about_file_path"] + afp_parent = posixpath.dirname(afp) + afp_parent = "/" + afp_parent if not afp_parent.startswith("/") else afp_parent + + # Update the 'about_resource' field with the relative path + # from the output location + if "about_resource" in ad.keys(): + about_resource = ad["about_resource"] + for resource in about_resource: + updated_about_resource = posixpath.normpath( + posixpath.join(afp_parent, resource) + ) + if resource == ".": + if not updated_about_resource == "/": + updated_about_resource = updated_about_resource + "/" + ad["about_resource"] = dict([(updated_about_resource, None)]) + del ad["about_file_path"] + serialized.append(ad) return serialized @@ -1235,121 +1900,383 @@ def write_output(abouts, location, format): # NOQA Return a list of Error objects. """ about_dicts = about_object_to_list_of_dictionary(abouts) - location = add_unc(location) - if format == 'csv': - errors = save_as_csv(location, about_dicts, get_field_names(abouts)) + if not location == "-": + location = add_unc(location) + if format == "csv": + save_as_csv(location, about_dicts, get_field_names(abouts)) + elif format == "json": + save_as_json(location, about_dicts) else: - errors = save_as_json(location, about_dicts) - return errors + save_as_excel(location, about_dicts) def save_as_json(location, about_dicts): - mode = 'w' - if python2: - mode = 'wb' - with io.open(location, mode=mode) as output_file: - data = util.format_about_dict_for_json_output(about_dicts) - output_file.write(json.dumps(data, indent=2)) - return [] + """ + Save the given data as a JSON file or print it to standard output. + """ + data = util.format_about_dict_for_json_output(about_dicts) + if location == "-": + json.dump(data, sys.stdout, indent=2) + else: + with open(location, mode="w") as output_file: + output_file.write(json.dumps(data, indent=2)) def save_as_csv(location, about_dicts, field_names): - errors = [] - with io.open(location, mode='w', encoding='utf-8', newline='') as output_file: - writer = csv.DictWriter(output_file, field_names) + """ + Save the given data as a CSV file or print it to standard output. + """ + if location == "-": + writer = csv.DictWriter(sys.stdout, field_names) writer.writeheader() - csv_formatted_list = util.format_about_dict_for_csv_output(about_dicts) + csv_formatted_list = util.format_about_dict_output(about_dicts) for row in csv_formatted_list: - # See https://github.com/dejacode/about-code-tool/issues/167 - try: + writer.writerow(row) + else: + with open( + location, mode="w", encoding="utf-8", newline="", errors="replace" + ) as output_file: + writer = csv.DictWriter(output_file, field_names) + writer.writeheader() + csv_formatted_list = util.format_about_dict_output(about_dicts) + for row in csv_formatted_list: writer.writerow(row) - except Exception as e: - msg = u'Generation skipped for ' + row['about_file_path'] + u' : ' + str(e) - errors.append(Error(CRITICAL, msg)) - return errors -def pre_process_and_fetch_license_dict(abouts, api_url, api_key): +def save_as_excel(location, about_dicts): """ - Modify a list of About data dictionaries by adding license information - fetched from the DejaCode API. + Save the given data as a Excel file. + """ + formatted_list = util.format_about_dict_output(about_dicts) + write_excel(location, formatted_list) + + +def pre_process_and_fetch_license_dict( + abouts, from_check=False, api_url=None, api_key=None, scancode=False, reference=None +): + """ + Return a dictionary containing the license information (key, name, text, url) + fetched from the ScanCode LicenseDB or DejaCode API. """ - dje_uri = urlparse(api_url) - domain = '{uri.scheme}://{uri.netloc}/'.format(uri=dje_uri) - dje_lic_urn = urljoin(domain, 'urn/?urn=urn:dje:license:') key_text_dict = {} captured_license = [] errors = [] + if api_url: + dje_uri = urlparse(api_url) + domain = "{uri.scheme}://{uri.netloc}/".format(uri=dje_uri) + lic_urn = urljoin(domain, "urn/?urn=urn:dje:license:") + url = api_url + else: + url = "https://scancode-licensedb.aboutcode.org/" if util.have_network_connection(): - if not valid_api_url(api_url): - msg = u"URL not reachable. Invalid '--api_url'. License generation is skipped." + if not valid_api_url(url): + msg = "URL not reachable. Invalid 'URL. License generation is skipped." errors.append(Error(ERROR, msg)) else: - msg = u'Network problem. Please check your Internet connection. License generation is skipped.' + msg = ( + "Network problem. Please check your Internet connection. License generation is skipped." + ) errors.append(Error(ERROR, msg)) + + if errors: + return key_text_dict, errors + + spdx_sclickey_dict = get_spdx_key_and_lic_key_from_licdb() for about in abouts: - # No need to go through all the about objects for license extraction if we detected - # invalid '--api_key' - auth_error = Error(ERROR, u"Authorization denied. Invalid '--api_key'. License generation is skipped.") + # No need to go through all the about objects if '--api_key' is invalid + auth_error = Error( + ERROR, "Authorization denied. Invalid '--api_key'. License generation is skipped." + ) if auth_error in errors: break - if about.license_expression.present: - special_char_in_expression, lic_list = parse_license_expression(about.license_expression.value) - if special_char_in_expression: - msg = (u"The following character(s) cannot be in the licesne_expression: " + - str(special_char_in_expression)) + + if scancode: + lic_exp = "" + lic_list = [] + if about.detected_license_expression.value: + lic_exp = about.detected_license_expression.value + about.license_expression.value = lic_exp + about.license_expression.present = True + + afp = "" + if about.about_file_path: + afp = about.about_file_path + + if not about.license_expression.value and about.spdx_license_expression.value: + lic_exp_value = "" + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + about.spdx_license_expression.value + ) + if special_char_in_expression or invalid_lic_exp: + if special_char_in_expression: + if afp: + msg = ( + afp + + ": The following character(s) cannot be in the spdx_license_expression: " + + str(special_char_in_expression) + ) + else: + msg = ( + "The following character(s) cannot be in the spdx_license_expression: " + + str(special_char_in_expression) + ) + else: + if afp: + msg = ( + afp + + ": This spdx_license_expression is invalid: " + + str(invalid_lic_exp) + ) + else: + msg = "This spdx_license_expression is invalid: " + str(invalid_lic_exp) errors.append(Error(ERROR, msg)) else: - for lic_key in lic_list: - if not lic_key in captured_license: - detail_list = [] - license_name, license_key, license_text, errs = api.get_license_details_from_api(api_url, api_key, lic_key) - for e in errs: - if e not in errors: - errors.append(e) - if license_key: - captured_license.append(lic_key) - dje_lic_url = dje_lic_urn + license_key - detail_list.append(license_name) - detail_list.append(license_text) - detail_list.append(dje_lic_url) - key_text_dict[license_key] = detail_list + spdx_lic_exp_segment = about.spdx_license_expression.value.split() + for spdx_lic_key in spdx_lic_exp_segment: + if lic_exp_value: + lic_exp_value = ( + lic_exp_value + + " " + + convert_spdx_expression_to_lic_expression( + spdx_lic_key, spdx_sclickey_dict + ) + ) + else: + lic_exp_value = convert_spdx_expression_to_lic_expression( + spdx_lic_key, spdx_sclickey_dict + ) + if lic_exp_value: + about.license_expression.value = lic_exp_value + about.license_expression.present = True + + lic_exp_list = [] + + if about.declared_license_expression.value: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + about.declared_license_expression.value + ) + if special_char_in_expression: + if afp: + msg = ( + afp + + ": The following character(s) cannot be in the declared_license_expression: " + + str(special_char_in_expression) + ) + else: + msg = ( + "The following character(s) cannot be in the declared_license_expression: " + + str(special_char_in_expression) + ) + errors.append(Error(ERROR, msg)) + if invalid_lic_exp: + if afp: + msg = ( + afp + + ": This declared_license_expression is invalid: " + + str(invalid_lic_exp) + ) + else: + msg = "This declared_license_expression is invalid: " + str(invalid_lic_exp) + errors.append(Error(ERROR, msg)) + if lic_list: + lic_exp_list.extend(lic_list) + + if about.other_license_expression.value: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + about.other_license_expression.value + ) + if special_char_in_expression: + if afp: + msg = ( + afp + + ": The following character(s) cannot be in the other_license_expression: " + + str(special_char_in_expression) + ) + else: + msg = "This declared_license_expression is invalid: " + str(invalid_lic_exp) + errors.append(Error(ERROR, msg)) + if invalid_lic_exp: + if afp: + msg = ( + afp + ": This other_license_expression is invalid: " + str(invalid_lic_exp) + ) + else: + msg = "This other_license_expression is invalid: " + str(invalid_lic_exp) + errors.append(Error(ERROR, msg)) + if lic_list: + lic_exp_list.extend(lic_list) + + if about.license_expression.value: + special_char_in_expression, lic_list, invalid_lic_exp = parse_license_expression( + about.license_expression.value + ) + if special_char_in_expression: + if afp: + msg = ( + afp + + ": The following character(s) cannot be in the license_expression: " + + str(special_char_in_expression) + ) + else: + msg = "The following character(s) cannot be in the license_expression: " + str( + special_char_in_expression + ) + errors.append(Error(ERROR, msg)) + if invalid_lic_exp: + if afp: + msg = afp + ": This license_expression is invalid: " + str(invalid_lic_exp) + else: + msg = "This license_expression is invalid: " + str(invalid_lic_exp) + errors.append(Error(ERROR, msg)) + if lic_list: + lic_exp_list.extend(lic_list) + if not about.license_key.value: + about.license_key.value = lic_list + + if lic_exp_list: + for lic_key in lic_exp_list: + if not lic_key in captured_license: + lic_url = "" + license_name = "" + license_filename = "" + license_text = "" + spdx_license_key = "" + detail_list = [] + captured_license.append(lic_key) + if api_key: + license_data, errs = api.get_license_details_from_api(url, api_key, lic_key) + # Catch incorrect API URL + if errs: + _, msg = errs[0] + if msg == "Invalid '--api_url'. License generation is skipped.": + errors.extend(errs) + return key_text_dict, errors + for severity, message in errs: + msg = afp + ": " + message + errors.append(Error(severity, msg)) + # We don't want to actually get the license information from the + # check utility + if from_check: + continue + if not license_data: + continue + license_name = license_data.get("short_name", "") + license_text = license_data.get("full_text", "") + spdx_license_key = license_data.get("spdx_license_key", "") + license_filename = lic_key + ".LICENSE" + lic_url = lic_urn + lic_key + else: + license_url = url + lic_key + ".json" + license_text_url = url + lic_key + ".LICENSE" + try: + response = head(license_url) + if response.status_code < 400: + json_url_content = get(license_url).text + # We don't want to actually get the license + # information from the check utility + if from_check: + continue + data = json.loads(json_url_content) + license_name = data["short_name"] + license_text = get(license_text_url).text + license_filename = data["key"] + ".LICENSE" + lic_url = url + license_filename + spdx_license_key = data["spdx_license_key"] + else: + if afp: + msg = afp + " : Invalid 'license': " + lic_key + else: + msg = "Invalid 'license': " + lic_key + errors.append(Error(ERROR, msg)) + continue + except exceptions.RequestException as e: + msg = f"An error occurred while trying to access the URL: {e}" + errors.append(Error(ERROR, msg)) + if not from_check: + detail_list.append(license_name) + detail_list.append(license_filename) + detail_list.append(license_text) + detail_list.append(lic_url) + detail_list.append(spdx_license_key) + key_text_dict[lic_key] = detail_list return key_text_dict, errors +def convert_spdx_expression_to_lic_expression(spdx_key, spdx_lic_dict): + """ + Translate the spdx_license_expression to license_expression and return + errors if spdx_license_key is not matched + """ + value = "" + if spdx_key in spdx_lic_dict: + value = spdx_lic_dict[spdx_key] + else: + if spdx_key.startswith("("): + mod_key = spdx_key.partition("(")[2] + value = "(" + convert_spdx_expression_to_lic_expression(mod_key, spdx_lic_dict) + elif spdx_key.endswith(")"): + mod_key = spdx_key.rpartition(")")[0] + value = convert_spdx_expression_to_lic_expression(mod_key, spdx_lic_dict) + ")" + else: + # This can be operator or key that don't have match + value = spdx_key + return value + + def parse_license_expression(lic_expression): licensing = Licensing() lic_list = [] - special_char = special_char_in_license_expresion(lic_expression) + invalid_lic_exp = "" + special_char = detect_special_char(lic_expression) if not special_char: # Parse the license expression and save it into a list - lic_list = licensing.license_keys(lic_expression) - return special_char, lic_list + try: + lic_list = licensing.license_keys(lic_expression) + except: + invalid_lic_exp = lic_expression + return special_char, lic_list, invalid_lic_exp -def special_char_in_license_expresion(lic_expression): +def detect_special_char(expression): not_support_char = [ - '!', '@', '#', '$', '%', '^', '&', '*', '=', '{', '}', - '|', '[', ']', '\\', ':', ';', '<', '>', '?', ',', '/'] + "!", + "@", + "#", + "$", + "^", + "&", + "*", + "=", + "{", + "}", + "|", + "[", + "]", + "\\", + ":", + ";", + "<", + ">", + "?", + ",", + "/", + ] special_character = [] for char in not_support_char: - if char in lic_expression: + if char in expression: special_character.append(char) return special_character def valid_api_url(api_url): try: - request = Request(api_url) - # This will always goes to exception as no key are provided. - # The purpose of this code is to validate the provided api_url is correct - urlopen(request) - except HTTPError as http_e: - # The 403 error code is refer to "Authentication credentials were not provided.". - # This is correct as no key are provided. - if http_e.code == 403: + response = get(api_url) + # The 403 error code is expected if the api_url is pointing to DJE as no + # API key is provided. The 200 status code represent connection success + # to scancode's LicenseDB. All other exception yield to invalid api_url + if response.status_code == 403 or response.status_code == 200: return True + else: + return False except: - # All other exceptions yield to invalid api_url - pass - return False + return False diff --git a/src/attributecode/templates/default_html.template b/src/attributecode/templates/default_html.template new file mode 100644 index 00000000..c7c31b65 --- /dev/null +++ b/src/attributecode/templates/default_html.template @@ -0,0 +1,84 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    {{ vartext['subtitle'] }}

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + {% for about_object in abouts %} +

    {{ about_object.name.value }}{% if about_object.version.value %} {{ about_object.version.value }}{% endif %}

    + {% endfor %} +
    + +
    + + {% for about_object in abouts %} +
    +

    {{ about_object.name.value }} {% if about_object.version.value %}{{ about_object.version.value }}{% endif %}

    + {% if about_object.license_expression.value %} +

    This component is licensed under {{ about_object.license_expression.value }}

    + {% endif %} + {% if about_object.copyright.value %} + 
    Copyright: {{about_object.copyright.value}}
    + {% endif %} + {% if about_object.notice_file.value %} + {% for notice in about_object.notice_file.value %} + 
    +                    {{ about_object.notice_file.value[notice] }}
+
    + {% endfor %} + {% endif %} + {% if about_object.license_key.value %} + {% for license_key in about_object.license_key.value %} + {% if license_key in common_licenses %} +

    Full text of {{ license_key }} is available at the end of this document.

    + {% else %} + {% for license in licenses_list %} + {% if license_key == license.key %} +

    {{ license.key }}

    + 
    {{ license.text | e }}
    + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% else %} + {% if about_object.license_file.value %} + {% for lic_file_name in about_object.license_file.value %} + {% if about_object.license_file.value[lic_file_name] %} + 
     {{ about_object.license_file.value[lic_file_name] | e}}
    + {% endif %} + {% endfor %} + {% endif %} + {% endif %} +
    + {% endfor %} + +
    + +

    Common Licenses Used in This Product

    + {% for license in licenses_list %} + {% if license.key in common_licenses %} +

    {{ license.key }}

    + 
     {{ license.text | e }}
    + {% endif %} + {% endfor %} + +

    End

    + + This file was generated with AttributeCode version: {{ tkversion }} on: {{ utcnow }} (UTC) + + + diff --git a/templates/default_json.template b/src/attributecode/templates/default_json.template similarity index 100% rename from templates/default_json.template rename to src/attributecode/templates/default_json.template diff --git a/src/attributecode/templates/license_ref.template b/src/attributecode/templates/license_ref.template new file mode 100644 index 00000000..1d0a5395 --- /dev/null +++ b/src/attributecode/templates/license_ref.template @@ -0,0 +1,69 @@ + + + + + {{ vartext['title'] }} + + +

    {{ vartext['title'] }}

    + + +
    +
    + {% for license in licenses_list %} +

    + {{ license.name }} + +

    + {% endfor %} + +
    +
    + + {% for license in licenses_list %} +
    +

    {{ license.name }}

    +

    This product contains the following open source software packages licensed under the terms of the license: {{license.name}}

    + +
    + {%for about_object in abouts %} + {% if loop.first %} + {% if license.url %} +

    License Gallery URL: {{license.url}}

    + {% endif %} + {% endif %} + {% if license.key in about_object.license_key.value %} +
  • {{ about_object.name.value }}{% if about_object.version.value %} - Version {{ about_object.version.value }}{% endif %}
    • + {% if about_object.copyright.value %} + 
    Copyright: {{about_object.copyright.value}}
    + {% endif %} + {% if about_object.notice_file.value %} + {% for notice in about_object.notice_file.value %} + 
    +                                {{ about_object.notice_file.value[notice] }}
+
    + {% endfor %} + {% endif %} + {% endif %} + {% if loop.last %} + 
    {{license.text}}
    + {% endif %} + {% endfor %} +
    +
    + {% endfor %} +
    +
    +

    End

    + + diff --git a/templates/list.csv b/src/attributecode/templates/list.csv similarity index 100% rename from templates/list.csv rename to src/attributecode/templates/list.csv diff --git a/src/attributecode/templates/scancode_html.template b/src/attributecode/templates/scancode_html.template new file mode 100644 index 00000000..d8582587 --- /dev/null +++ b/src/attributecode/templates/scancode_html.template @@ -0,0 +1,88 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    {{ vartext['subtitle'] }}

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + {% set index = namespace(value=0) %} + {% for about_object in abouts %} + {% set captured = {} %} + {% if about_object.license_key.value %} + {% if not captured[about_object.name.value] %} +

    {{ about_object.name.value }}{% if about_object.version.value %} {{ about_object.version.value }}{% endif %}

    + {% set _ = captured.update({ about_object.name.value: true }) %} + {% set index.value = index.value + 1 %} + {% endif %} + {% endif %} + {% endfor %} +
    + +
    + + {% set index = namespace(value=0) %} + {% for about_object in abouts %} + {% set captured = {} %} + {% if about_object.license_key.value %} + {% if not captured[about_object.name.value] %} +
    +

    {{ about_object.name.value }} {% if about_object.version.value %}{{ about_object.version.value }}{% endif %}

    + {% set _ = captured.update({ about_object.name.value: true }) %} + {% set index.value = index.value + 1 %} + {% endif %} + {% if about_object.copyrights.value %} + {% for copyright in about_object.copyrights.value %} + 
     {{ copyright['copyright'] }}
    + {% endfor %} + {% endif %} + + {% for lic_key_exp in about_object.license_key_expression.value %} +

    This component is licensed under {{ lic_key_exp }}

    + {% endfor %} + + {% for lic_key_exp in about_object.license_key.value %} + {% for lic_key in lic_key_exp %} + {% if lic_key in common_licenses %} +

    Full text of {{ lic_key }} is available at the end of this document.

    + {% else %} + {% for license in licenses_list %} + {% if lic_key == license.key %} +

    {{ license.key }}

    + 
     {{ license.text | e }}
    + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% endfor %} + {% endif %} +
    + {% endfor %} + +
    + +

    Common Licenses Used in This Product

    + {% for license in licenses_list %} + {% if license.key in common_licenses %} +

    {{ license.key }}

    + 
     {{ license.text | e }}
    + {% endif %} + {% endfor %} + +

    End

    + + This file was generated with AttributeCode version: {{ tkversion }} on: {{ utcnow }} (UTC) + + diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index ccfb1901..f0972f71 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,157 +13,217 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import Counter -from collections import OrderedDict -import io +import json +from collections import Counter, OrderedDict +from itertools import zip_longest import attr +import openpyxl from attributecode import CRITICAL from attributecode import Error from attributecode import saneyaml from attributecode.util import csv -from attributecode.util import python2 from attributecode.util import replace_tab_with_spaces -if python2: # pragma: nocover - from itertools import izip_longest as zip_longest # NOQA -else: # pragma: nocover - from itertools import zip_longest # NOQA - - -def transform_csv_to_csv(location, output, transformer): +def transform_csv(location): """ - Read a CSV file at `location` and write a new CSV file at `output`. Apply - transformations using the `transformer` Tranformer. - Return a list of Error objects. + Read a CSV file at `location` and convert data into list of dictionaries. """ - if not transformer: - raise ValueError('Cannot transform without Transformer') - + errors = [] + new_data = [] rows = read_csv_rows(location) + data = iter(rows) + names = next(rows) + field_names = strip_trailing_fields_csv(names) + dupes = check_duplicate_fields(field_names) - column_names, data, errors = transform_data(rows, transformer) + if dupes: + msg = "Duplicated field name: %(name)s" + for name in dupes: + errors.append(Error(CRITICAL, msg % locals())) - if errors: - return errors - else: - write_csv(output, data, column_names) - return [] + if not errors: + # Convert to dicts + new_data = [dict(zip_longest(field_names, item)) for item in data] + + return new_data, errors -def transform_data(rows, transformer): +def transform_json(location): """ - Read a list of list of CSV-like data `rows` and apply transformations using the - `transformer` Tranformer. - Return a tuple of: - ([column names...], [transformed ordered dict...], [Error objects..]) + Read a JSON file at `location` and convert data into list of dictionaries. """ + errors = [] + new_data = [] + items = read_json(location) + data = normalize_dict_data(items) + new_data = strip_trailing_fields_json(data) - if not transformer: - return rows + return new_data, errors + +def transform_excel(location, worksheet=None): + """ + Read a XLSX file at `location` and convert data into list of dictionaries. + """ errors = [] - rows = iter(rows) - column_names = next(rows) - column_names = transformer.clean_columns(column_names) + new_data = [] + dupes, new_data = read_excel(location, worksheet) + if dupes: + msg = "Duplicated field name: %(name)s" + for name in dupes: + errors.append(Error(CRITICAL, msg % locals())) + return new_data, errors - dupes = check_duplicate_columns(column_names) - if dupes: - msg = 'Duplicated column name: {name}' - errors.extend(Error(CRITICAL, msg.format(name)) for name in dupes) - return column_names, [], errors +def strip_trailing_fields_csv(names): + """ + Strip trailing spaces for field names #456 + """ + field_names = [] + for name in names: + field_names.append(name.strip()) + return field_names - column_names = transformer.apply_renamings(column_names) - # convert to dicts using the renamed columns - data = [OrderedDict(zip_longest(column_names, row)) for row in rows] +def strip_trailing_fields_json(items): + """ + Strip trailing spaces for field name #456 + """ + data = [] + for item in items: + od = {} + for field in item: + stripped_field_name = field.strip() + od[stripped_field_name] = item[field] + data.append(od) + return data - if transformer.column_filters: - data = list(transformer.filter_columns(data)) - column_names = [c for c in column_names if c in transformer.column_filters] - errors = transformer.check_required_columns(data) - if errors: - return column_names, data, errors +def normalize_dict_data(data): + """ + Check if the input data from scancode-toolkit and normalize to a normal + dictionary if it is. + Return a list type of normalized dictionary. + """ + try: + # Check if this is a JSON output from scancode-toolkit + if data["headers"][0]["tool_name"] == "scancode-toolkit": + # only takes data inside "files" + new_data = data["files"] + except: + new_data = data + if not isinstance(new_data, list): + new_data = [new_data] + return new_data + + +def transform_data(data, transformer): + """ + Read a dictionary and apply transformations using the + `transformer` Transformer. + Return a tuple of: + ([field names...], [transformed ordered dict...], [Error objects..]) + """ + renamed_field_data = transformer.apply_renamings(data) + + if transformer.field_filters: + renamed_field_data = list(transformer.filter_fields(renamed_field_data)) + + if transformer.exclude_fields: + renamed_field_data = list(transformer.filter_excluded(renamed_field_data)) - return column_names, data, errors + errors = transformer.check_required_fields(renamed_field_data) + if errors: + return data, errors + return renamed_field_data, errors -tranformer_config_help = ''' +tranformer_config_help = """ A transform configuration file is used to describe which transformations and validations to apply to a source CSV file. This is a simple text file using YAML format, using the same format as an .ABOUT file. The attributes that can be set in a configuration file are: -* column_renamings: -An optional map of source CSV column name to target CSV new column name that -is used to rename CSV columns. +* field_renamings: +An optional map of source CSV or JSON field name to target CSV/JSON new field name that +is used to rename CSV fields. -For instance with this configuration the columns "Directory/Location" will be +For instance with this configuration the fields "Directory/Location" will be renamed to "about_resource" and "foo" to "bar": - column_renamings: - 'Directory/Location' : about_resource - foo : bar + field_renamings: + about_resource : 'Directory/Location' + bar : foo The renaming is always applied first before other transforms and checks. All -other column names referenced below are these that exist AFTER the renamings -have been applied to the existing column names. +other field names referenced below are these that exist AFTER the renamings +have been applied to the existing field names. -* required_columns: -An optional list of required column names that must have a value, beyond the -standard columns names. If a source CSV does not have such a column or a row is -missing a value for a required column, an error is reported. +* required_fields: +An optional list of required field names that must have a value, beyond the +standard fields names. If a source CSV/JSON does not have such a field or a row is +missing a value for a required field, an error is reported. -For instance with this configuration an error will be reported if the columns +For instance with this configuration an error will be reported if the fields "name" and "version" are missing or if any row does not have a value set for -these columns: - required_columns: +these fields: + required_fields: - name - version -* column_filters: -An optional list of column names that should be kept in the transformed CSV. If -this list is provided, all the columns from the source CSV that should be kept -in the target CSV must be listed be even if they are standard or required -columns. If this list is not provided, all source CSV columns are kept in the -transformed target CSV. +* field_filters: +An optional list of field names that should be kept in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be kept +in the target CSV/JSON must be listed regardless of either standard or required +fields. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. -For instance with this configuration the target CSV will only contains the "name" -and "version" columns and no other column: - column_filters: +For instance with this configuration the target CSV/JSON will only contains the "name" +and "version" fields and no other field: + field_filters: - name - version -''' + +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp +""" @attr.attributes class Transformer(object): __doc__ = tranformer_config_help - column_renamings = attr.attrib(default=attr.Factory(dict)) - required_columns = attr.attrib(default=attr.Factory(list)) - column_filters = attr.attrib(default=attr.Factory(list)) + field_renamings = attr.attrib(default=attr.Factory(dict)) + required_fields = attr.attrib(default=attr.Factory(list)) + field_filters = attr.attrib(default=attr.Factory(list)) + exclude_fields = attr.attrib(default=attr.Factory(list)) - # a list of all the standard columns from AboutCode toolkit - standard_columns = attr.attrib(default=attr.Factory(list), init=False) - # a list of the subset of standard columns that are essential and MUST be + # a list of all the standard fields from AboutCode toolkit + standard_fields = attr.attrib(default=attr.Factory(list), init=False) + # a list of the subset of standard fields that are essential and MUST be # present for AboutCode toolkit to work - essential_columns = attr.attrib(default=attr.Factory(list), init=False) + essential_fields = attr.attrib(default=attr.Factory(list), init=False) # called by attr after the __init__() def __attrs_post_init__(self, *args, **kwargs): from attributecode.model import About + about = About() - self.essential_columns = list(about.required_fields) - self.standard_columns = [f.name for f in about.all_fields()] + self.essential_fields = list(about.required_fields) + self.standard_fields = [f.name for f in about.all_fields()] @classmethod def default(cls): @@ -171,9 +231,10 @@ def default(cls): Return a default Transformer with built-in transforms. """ return cls( - column_renamings={}, - required_columns=[], - column_filters=[], + field_renamings={}, + required_fields=[], + field_filters=[], + exclude_fields=[], ) @classmethod @@ -182,21 +243,22 @@ def from_file(cls, location): Load and return a Transformer instance from a YAML configuration file at `location`. """ - with io.open(location, encoding='utf-8') as conf: + with open(location, encoding="utf-8", errors="replace") as conf: data = saneyaml.load(replace_tab_with_spaces(conf.read())) return cls( - column_renamings=data.get('column_renamings', {}), - required_columns=data.get('required_columns', []), - column_filters=data.get('column_filters', []), + field_renamings=data.get("field_renamings", {}), + required_fields=data.get("required_fields", []), + field_filters=data.get("field_filters", []), + exclude_fields=data.get("exclude_fields", []), ) - def check_required_columns(self, data): + def check_required_fields(self, data): """ Return a list of Error for a `data` list of ordered dict where a - dict is missing a value for a required column name. + dict is missing a value for a required field name. """ errors = [] - required = set(self.essential_columns + self.required_columns) + required = set(self.essential_fields + self.required_fields) if not required: return [] @@ -205,73 +267,178 @@ def check_required_columns(self, data): if not missings: continue - missings = ', '.join(missings) - msg = 'Row {rn} is missing required values for columns: {missings}' + missings = ", ".join(missings) + msg = "Row {rn} is missing required values for fields: {missings}" errors.append(Error(CRITICAL, msg.format(**locals()))) + return errors - def apply_renamings(self, column_names): + def apply_renamings(self, data): """ - Return a tranformed list of `column_names` where columns are renamed + Return a tranformed list of `field_names` where fields are renamed based on this Transformer configuration. """ - renamings = self.column_renamings + renamings = self.field_renamings + renamed_to_list = list(renamings.keys()) + renamed_from_list = list(renamings.values()) if not renamings: - return column_names - renamings = {n.lower(): rn.lower() for n, rn in renamings.items()} + return data + if isinstance(data, dict): + renamed_obj = {} + for key, value in data.items(): + if key in renamed_from_list: + for idx, renamed_from_key in enumerate(renamed_from_list): + if key == renamed_from_key: + renamed_key = renamed_to_list[idx] + renamed_obj[renamed_key] = self.apply_renamings(value) + else: + renamed_obj[key] = self.apply_renamings(value) + return renamed_obj + elif isinstance(data, list): + return [self.apply_renamings(item) for item in data] + else: + return data - renamed = [] - for name in column_names: - name = name.lower() - new_name = renamings.get(name, name) - renamed.append(new_name) - return renamed + """ + def clean_fields(self, field_names): - def clean_columns(self, column_names): - """ - Apply standard cleanups to a list of columns and return these. - """ - if not column_names: - return column_names - return [c.strip().lower() for c in column_names] + Apply standard cleanups to a list of fields and return these. - def filter_columns(self, data): + if not field_names: + return field_names + return [c.strip().lower() for c in field_names] + """ + + def filter_fields(self, data): """ Yield transformed dicts from a `data` list of dicts keeping only - columns with a name in the `column_filters`of this Transformer. - Return the data unchanged if no `column_filters` exists. + fields with a name in the `field_filters`of this Transformer. + Return the data unchanged if no `field_filters` exists. """ - column_filters = set(self.clean_columns(self.column_filters)) + # field_filters = set(self.clean_fields(self.field_filters)) + field_filters = set(self.field_filters) for entry in data: - items = ((k, v) for k, v in entry.items() if k in column_filters) - yield OrderedDict(items) + yield {k: v for k, v in entry.items() if k in field_filters} - -def check_duplicate_columns(column_names): + def filter_excluded(self, data): + """ + Yield transformed dicts from a `data` list of dicts excluding + fields with names in the `exclude_fields`of this Transformer. + Return the data unchanged if no `exclude_fields` exists. + """ + # exclude_fields = set(self.clean_fields(self.exclude_fields)) + exclude_fields = set(self.exclude_fields) + filtered_list = [] + for entry in data: + result = {} + for k, v in entry.items(): + if type(v) == list: + result[k] = self.filter_excluded(v) + elif k not in exclude_fields: + result[k] = v + filtered_list.append(result) + # yield result + # yield {k: v for k, v in entry.items() if k not in exclude_fields} + return filtered_list + + +def check_duplicate_fields(field_names): """ - Check that there are no duplicate in the `column_names` list of column name - strings, ignoring case. Return a list of unique duplicated column names. + Check that there are no duplicate in the `field_names` list of field name + strings, ignoring case. Return a list of unique duplicated field names. """ - counted = Counter(c.lower() for c in column_names) - return [column for column, count in sorted(counted.items()) if count > 1] + counted = Counter(c.lower() for c in field_names) + return [field for field, count in sorted(counted.items()) if count > 1] def read_csv_rows(location): """ Yield rows (as a list of values) from a CSV file at `location`. """ - with io.open(location, encoding='utf-8', errors='replace') as csvfile: + with open(location, encoding="utf-8", errors="replace") as csvfile: reader = csv.reader(csvfile) for row in reader: yield row -def write_csv(location, data, column_names): # NOQA +def read_json(location): """ - Write a CSV file at `location` the `data` list of ordered dicts using the - `column_names`. + Yield rows (as a list of values) from a CSV file at `location`. """ - with io.open(location, 'w', encoding='utf-8', newline='\n') as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=column_names) + with open(location, encoding="utf-8", errors="replace") as jsonfile: + return json.load(jsonfile) + + +def write_csv(location, data): + """ + Write a CSV file at `location` with the `data` which is a list of ordered dicts. + """ + field_names = list(data[0].keys()) + with open(location, "w", encoding="utf-8", newline="\n", errors="replace") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=field_names) writer.writeheader() writer.writerows(data) + + +def write_json(location, data): + """ + Write a JSON file at `location` the `data` list of ordered dicts. + """ + with open(location, "w") as jsonfile: + json.dump(data, jsonfile, indent=3) + + +def read_excel(location, worksheet=None): + """ + Read XLSX at `location`, return a list of ordered dictionaries, one + for each row. + """ + results = [] + errors = [] + input_bom = openpyxl.load_workbook(location) + if worksheet: + sheet_obj = input_bom[worksheet] + else: + sheet_obj = input_bom.active + max_col = sheet_obj.max_column + + index = 1 + col_keys = [] + mapping_dict = {} + while index <= max_col: + value = sheet_obj.cell(row=1, column=index).value + if value in col_keys: + msg = "Duplicated column name, " + str(value) + ", detected." + errors.append(Error(CRITICAL, msg)) + return errors, results + if value in mapping_dict: + value = mapping_dict[value] + col_keys.append(value) + index = index + 1 + + for row in sheet_obj.iter_rows(min_row=2, values_only=True): + row_dict = OrderedDict() + index = 0 + while index < max_col: + value = row[index] + if value: + row_dict[col_keys[index]] = value + else: + row_dict[col_keys[index]] = "" + index = index + 1 + results.append(row_dict) + return errors, results + + +def write_excel(location, data): + wb = openpyxl.Workbook() + ws = wb.active + + # Get the header + headers = list(data[0].keys()) + ws.append(headers) + + for elements in data: + ws.append([elements.get(h) for h in headers]) + + wb.save(location) diff --git a/src/attributecode/util.py b/src/attributecode/util.py index a3abe6ee..d63e6b6a 100644 --- a/src/attributecode/util.py +++ b/src/attributecode/util.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2013-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,50 +13,41 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals +from collections import OrderedDict import codecs -from collections import OrderedDict +import csv import json import ntpath +import openpyxl import os import posixpath import re import shutil import string import sys +from distutils.dir_util import copy_tree +from itertools import zip_longest from attributecode import CRITICAL from attributecode import WARNING from attributecode import Error - -python2 = sys.version_info[0] < 3 - -if python2: # pragma: nocover - from itertools import izip_longest as zip_longest # NOQA -else: # pragma: nocover - from itertools import zip_longest # NOQA - -if python2: # pragma: nocover - from backports import csv # NOQA - # monkey patch backports.csv until bug is fixed - # https://github.com/ryanhiebert/backports.csv/issues/30 - csv.dict = OrderedDict -else: # pragma: nocover - import csv # NOQA - - -on_windows = 'win32' in sys.platform +on_windows = "win32" in sys.platform # boolean field name -boolean_fields = ['redistribute', 'attribute', 'track_change', 'modified', 'internal_use_only'] -file_fields = ['about_resource', 'notice_file', 'changelog_file', 'author_file'] +boolean_fields = [ + "redistribute", + "attribute", + "track_change", + "modified", + "internal_use_only", +] +file_fields = ["about_resource", "notice_file", "changelog_file", "author_file"] + def to_posix(path): - """ + r""" Return a path using the posix path separator given a path that may contain posix or windows separators, converting "\\" to "/". NB: this path will still be valid in the windows explorer (except for a UNC or share name). It @@ -66,11 +57,15 @@ def to_posix(path): return path.replace(ntpath.sep, posixpath.sep) -UNC_PREFIX = u'\\\\?\\' +UNC_PREFIX = "\\\\?\\" UNC_PREFIX_POSIX = to_posix(UNC_PREFIX) -UNC_PREFIXES = (UNC_PREFIX_POSIX, UNC_PREFIX,) +UNC_PREFIXES = ( + UNC_PREFIX_POSIX, + UNC_PREFIX, +) -valid_file_chars = string.digits + string.ascii_letters + '_-.+()~[]{}|' + ' ' +valid_file_chars = "_-.+()~[]{}@%!$," +invalid_file_chars = string.punctuation.translate(str.maketrans("", "", valid_file_chars)) def invalid_chars(path): @@ -80,7 +75,8 @@ def invalid_chars(path): path = to_posix(path) rname = resource_name(path) name = rname.lower() - return [c for c in name if c not in valid_file_chars] + + return [c for c in name if c in invalid_file_chars] def check_file_names(paths): @@ -89,11 +85,6 @@ def check_file_names(paths): there are no case-insensitive duplicates in any given directories. Return a list of errors. - From spec : - A file name can contain only these US-ASCII characters: - - digits from 0 to 9 - - uppercase and lowercase letters from A to Z - - the _ underscore, - dash and . period signs. From spec: The case of a file name is not significant. On case-sensitive file systems (such as Linux), a tool must raise an error if two ABOUT files @@ -106,9 +97,8 @@ def check_file_names(paths): path = orig_path invalid = invalid_chars(path) if invalid: - invalid = ''.join(invalid) - msg = ('Invalid characters %(invalid)r in file name at: ' - '%(path)r' % locals()) + invalid = "".join(invalid) + msg = "Invalid characters %(invalid)r in file name at: %(path)r" % locals() errors.append(Error(CRITICAL, msg)) path = to_posix(orig_path) @@ -119,37 +109,42 @@ def check_file_names(paths): path = posixpath.abspath(path) existing = seen.get(path) if existing: - msg = ('Duplicate files: %(orig_path)r and %(existing)r ' - 'have the same case-insensitive file name' % locals()) + msg = ( + "Duplicate files: %(orig_path)r and %(existing)r " + "have the same case-insensitive file name" % locals() + ) errors.append(Error(CRITICAL, msg)) else: seen[path] = orig_path return errors + def wrap_boolean_value(context): - updated_context = '' + updated_context = "" for line in context.splitlines(): """ wrap the boolean value in quote """ - key = line.partition(':')[0] - value = line.partition(':')[2].strip() + key = line.partition(":")[0] + value = line.partition(":")[2].strip() value = '"' + value + '"' if key in boolean_fields and not value == "": - updated_context += key + ': ' + value + '\n' + updated_context += key + ": " + value + "\n" else: - updated_context += line + '\n' + updated_context += line + "\n" return updated_context + def replace_tab_with_spaces(context): - updated_context = '' + updated_context = "" for line in context.splitlines(): """ Replace tab with 4 spaces """ - updated_context += line.replace('\t', ' ') + '\n' + updated_context += line.replace("\t", " ") + "\n" return updated_context + # TODO: rename to normalize_path def get_absolute(location): """ @@ -170,7 +165,8 @@ def get_locations(location): """ location = add_unc(location) location = get_absolute(location) - assert os.path.exists(location) + if not os.path.exists(location): + raise FileNotFoundError(f"Expected path does not exist: {location}") if os.path.isfile(location): yield location @@ -181,15 +177,100 @@ def get_locations(location): yield posixpath.join(bd, name) -def get_about_locations(location): +def get_about_locations(location, exclude=None): """ Return a list of locations of ABOUT files given the `location` of a a file or a directory tree containing ABOUT files. File locations are normalized using posix path separators. """ + pattern_characters_list = ["*", "?", "[", "!"] + import fnmatch + for loc in get_locations(location): - if is_about_file(loc): - yield loc + exclude_match = False + if exclude: + for item in exclude: + is_pattern = False + for character in pattern_characters_list: + if character in item: + is_pattern = True + break + exclude_path = posixpath.join(location, item) + normalized_excluded_path = posixpath.normpath( + add_unc(exclude_path).replace("\\", "/") + ) + # Since 'normpath' removes the trailing '/', it is necessary + # to append the '/' back for proper matching. + if not is_pattern and item.endswith("/"): + normalized_excluded_path += "/" + if is_pattern: + if fnmatch.fnmatch(loc, normalized_excluded_path): + exclude_match = True + break + else: + if normalized_excluded_path in loc: + exclude_match = True + break + if not exclude_match: + if is_about_file(loc): + yield loc + + +def norm(p): + """ + Normalize the path + """ + if p.startswith(UNC_PREFIX) or p.startswith(to_posix(UNC_PREFIX)): + p = p.strip(UNC_PREFIX).strip(to_posix(UNC_PREFIX)) + p = to_posix(p) + p = p.strip(posixpath.sep) + p = posixpath.normpath(p) + return p + + +def get_spdx_key_and_lic_key_from_licdb(): + """ + Return a dictionary list that fetch all licenses from licenseDB. The + "spdx_license_key" will be the key of the dictionary and the "license_key" + will be the value of the directionary + """ + import requests + + lic_dict = dict() + + # URL of the license index + url = "https://scancode-licensedb.aboutcode.org/index.json" + + """ + Sample of one of the license in the index.json + { + "license_key": "bsd-new", + "category": "Permissive", + "spdx_license_key": "BSD-3-Clause", + "other_spdx_license_keys": [ + "LicenseRef-scancode-libzip" + ], + "is_exception": false, + "is_deprecated": false, + "json": "bsd-new.json", + "yaml": "bsd-new.yml", + "html": "bsd-new.html", + "license": "bsd-new.LICENSE" + }, + """ + response = requests.get(url) + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Retrieve the JSON data from the response + licenses_index = response.json() + + for license in licenses_index: + lic_dict[license["spdx_license_key"]] = license["license_key"] + if license["other_spdx_license_keys"]: + for other_spdx in license["other_spdx_license_keys"]: + lic_dict[other_spdx] = license["license_key"] + + return lic_dict def get_relative_path(base_loc, full_loc): @@ -198,20 +279,11 @@ def get_relative_path(base_loc, full_loc): The first segment of the different between full_loc and base_loc will become the first segment of the returned path. """ - def norm(p): - if p.startswith(UNC_PREFIX) or p.startswith(to_posix(UNC_PREFIX)): - p = p.strip(UNC_PREFIX).strip(to_posix(UNC_PREFIX)) - p = to_posix(p) - p = p.strip(posixpath.sep) - p = posixpath.normpath(p) - return p - base = norm(base_loc) path = norm(full_loc) - assert path.startswith(base), ('Cannot compute relative path: ' - '%(path)r does not start with %(base)r' - % locals()) + if not path.startswith(base): + raise ValueError(f"Cannot compute relative path: {path!r} does not start with {base!r}") base_name = resource_name(base) no_dir = base == base_name same_loc = base == path @@ -226,7 +298,7 @@ def norm(p): parent_dir = resource_name(parent_dir) relative = posixpath.join(parent_dir, base_name) else: - relative = path[len(base) + 1:] + relative = path[len(base) + 1 :] # We don't want to keep the first segment of the root of the returned path. # See https://github.com/nexB/attributecode/issues/276 # relative = posixpath.join(base_name, relative) @@ -234,7 +306,7 @@ def norm(p): def to_native(path): - """ + r""" Return a path using the current OS path separator given a path that may contain posix or windows separators, converting "/" to "\\" on windows and "\\" to "/" on posix OSes. @@ -250,7 +322,7 @@ def is_about_file(path): """ if path: path = path.lower() - return path.endswith('.about') and path != '.about' + return path.endswith(".about") and path != ".about" def resource_name(path): @@ -264,21 +336,16 @@ def resource_name(path): return right.strip() - def load_csv(location): """ Read CSV at `location`, return a list of ordered dictionaries, one for each row. """ results = [] - # FIXME: why ignore encoding errors here? - with codecs.open(location, mode='rb', encoding='utf-8', - errors='ignore') as csvfile: + with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile: for row in csv.DictReader(csvfile): # convert all the column keys to lower case - updated_row = OrderedDict( - [(key.lower(), value) for key, value in row.items()] - ) + updated_row = {key.lower().strip(): value for key, value in row.items()} results.append(updated_row) return results @@ -288,84 +355,31 @@ def load_json(location): Read JSON file at `location` and return a list of ordered dicts, one for each entry. """ - # FIXME: IMHO we should know where the JSON is from and its shape - # FIXME use: object_pairs_hook=OrderedDict with open(location) as json_file: results = json.load(json_file) - # If the loaded JSON is not a list, - # - JSON output from AboutCode Manager: - # look for the "components" field as it is the field - # that contain everything the tool needs and ignore other fields. - # For instance, - # { - # "aboutcode_manager_notice":"xyz", - # "aboutcode_manager_version":"xxx", - # "components": - # [{ - # "license_expression":"apache-2.0", - # "copyright":"Copyright (c) 2017 nexB Inc.", - # "path":"ScanCode", - # ... - # }] - # } - # - # - JSON output from ScanCode: - # look for the "files" field as it is the field - # that contain everything the tool needs and ignore other fields: - # For instance, - # { - # "scancode_notice":"xyz", - # "scancode_version":"xxx", - # "files": - # [{ - # "path": "test", - # "type": "directory", - # "name": "test", - # ... - # }] - # } - # - # - JSON file that is not produced by scancode or aboutcode toolkit - # For instance, - # { - # "path": "test", - # "type": "directory", - # "name": "test", - # ... - # } - # FIXME: this is too clever and complex... IMHO we should not try to guess the format. - # instead a command line option should be provided explictly to say what is the format - if isinstance(results, list): - results = sorted(results) - else: - if u'aboutcode_manager_notice' in results: - results = results['components'] - elif u'scancode_notice' in results: - results = results['files'] - else: - results = [results] + if not isinstance(results, list): + results = [results] + return results # FIXME: rename to is_online: BUT do we really need this at all???? +# This is needed to check for the network connection when user wants to fetch +# the licenses from DJE/LicenseDB def have_network_connection(): """ Return True if an HTTP connection to some public web site is possible. """ - import socket - if python2: - import httplib # NOQA - else: - import http.client as httplib # NOQA + import requests - http_connection = httplib.HTTPConnection('dejacode.org', timeout=10) # NOQA - try: - http_connection.connect() - except socket.error: - return False - else: + url = "https://scancode-licensedb.aboutcode.org/" + + response = requests.get(url) + if response.status_code == 200: return True + else: + return False def extract_zip(location): @@ -377,10 +391,10 @@ def extract_zip(location): import tempfile if not zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) - archive_base_name = os.path.basename(location).replace('.zip', '') - base_dir = tempfile.mkdtemp(prefix='aboutcode-toolkit-extract-') + archive_base_name = os.path.basename(location).replace(".zip", "") + base_dir = tempfile.mkdtemp(prefix="aboutcode-toolkit-extract-") target_dir = os.path.join(base_dir, archive_base_name) target_dir = add_unc(target_dir) os.makedirs(target_dir) @@ -406,7 +420,7 @@ def extract_zip(location): if not os.path.exists(target): os.makedirs(add_unc(target)) if not os.path.exists(target): - with open(target, 'wb') as f: + with open(target, "wb") as f: f.write(content) return target_dir @@ -432,39 +446,109 @@ def copy_license_notice_files(fields, base_dir, reference_dir, afp): where reference license an notice files are stored and the `afp` about_file_path value, this function will copy to the base_dir the license_file or notice_file if found in the reference_dir - """ - lic_name = '' + errors = [] + copy_file_name = "" for key, value in fields: - if key == 'license_file' or key == 'notice_file': - lic_name = value + if key == "license_file" or key == "notice_file": + if value: + # This is to handle multiple license_file value in CSV format + # The following code will construct a list to contain the + # license file(s) that need to be copied. + # Note that *ONLY* license_file field allows \n. Others file + # fields that have \n will prompts error at validation stage + file_list = [] + if "\n" in value: + f_list = value.split("\n") + else: + if not isinstance(value, list): + f_list = [value] + else: + f_list = value + # The following code is to adopt the approach from #404 + # to use comma for multiple files which refer the same license + for item in f_list: + if "," in item: + item_list = item.split(",") + for i in item_list: + file_list.append(i.strip()) + else: + file_list.append(item) + else: + continue - from_lic_path = posixpath.join(to_posix(reference_dir), lic_name) - about_file_dir = os.path.dirname(to_posix(afp)).lstrip('/') - to_lic_path = posixpath.join(to_posix(base_dir), about_file_dir) + for copy_file_name in file_list: + from_lic_path = posixpath.join(to_posix(reference_dir), copy_file_name) + about_file_dir = os.path.dirname(to_posix(afp)).lstrip("/") + to_lic_path = posixpath.join(to_posix(base_dir), about_file_dir) + if not os.path.exists(posixpath.join(to_lic_path, copy_file_name)): + err = copy_file(from_lic_path, to_lic_path) + if err: + errors.append(err) + return errors - if on_windows: - from_lic_path = add_unc(from_lic_path) - to_lic_path = add_unc(to_lic_path) - # Strip the white spaces - from_lic_path = from_lic_path.strip() - to_lic_path = to_lic_path.strip() +def copy_file(from_path, to_path): + error = "" + # Return if the from_path is empty or None. + if not from_path: + return - # Errors will be captured when doing the validation - if not posixpath.exists(from_lic_path): - continue + if on_windows: + if not from_path.startswith(UNC_PREFIXES): + from_path = add_unc(from_path) + if not to_path.startswith(UNC_PREFIXES): + to_path = add_unc(to_path) - if not posixpath.exists(to_lic_path): - os.makedirs(to_lic_path) - try: - shutil.copy2(from_lic_path, to_lic_path) - except Exception as e: - print(repr(e)) - print('Cannot copy file at %(from_lic_path)r.' % locals()) + # Strip the white spaces + from_path = from_path.strip() + to_path = to_path.strip() + # Errors will be captured when doing the validation + if not os.path.exists(from_path): + return "" + + if not posixpath.exists(to_path): + os.makedirs(to_path) + try: + if os.path.isdir(from_path): + # Copy the whole directory structure + if from_path.endswith("/"): + from_path = from_path.rpartition("/")[0] + folder_name = os.path.basename(from_path) + to_path = os.path.join(to_path, folder_name) + if os.path.exists(to_path): + msg = to_path + " is already existed and is replaced by " + from_path + error = Error(WARNING, msg) + copy_tree(from_path, to_path) + else: + file_name = os.path.basename(from_path) + to_file_path = os.path.join(to_path, file_name) + if os.path.exists(to_file_path): + msg = to_file_path + " is already existed and is replaced by " + from_path + error = Error(WARNING, msg) + shutil.copy2(from_path, to_path) + return error + except Exception as e: + msg = "Cannot copy file at %(from_path)r." % locals() + error = Error(CRITICAL, msg) + return error + + +def ungroup_licenses_from_sctk(value): + # Return a list of dictionary with lic_key and score + # extracted from SCTK scan + detected_license_list = [] + for detected_license in value: + for lic in detected_license["matches"]: + lic_exp = lic["license_expression"] + score = lic["score"] + detected_license_list.append({"lic_exp": lic_exp, "score": score}) + return detected_license_list # FIXME: we should use a license object instead + + def ungroup_licenses(licenses): """ Ungroup multiple licenses information @@ -473,41 +557,58 @@ def ungroup_licenses(licenses): lic_name = [] lic_file = [] lic_url = [] + spdx_lic_key = [] + lic_score = [] + lic_matched_text = [] for lic in licenses: - if 'key' in lic: - lic_key.append(lic['key']) - if 'name' in lic: - lic_name.append(lic['name']) - if 'file' in lic: - lic_file.append(lic['file']) - if 'url' in lic: - lic_url.append(lic['url']) - return lic_key, lic_name, lic_file, lic_url + if "key" in lic: + lic_key.append(lic["key"]) + if "name" in lic: + lic_name.append(lic["name"]) + if "file" in lic: + lic_file.append(lic["file"]) + if "url" in lic: + lic_url.append(lic["url"]) + if "spdx_license_key" in lic: + spdx_lic_key.append(lic["spdx_license_key"]) + if "score" in lic: + lic_score.append(lic["score"]) + if "matched_text" in lic: + lic_matched_text.append(lic["matched_text"]) + return ( + lic_key, + lic_name, + lic_file, + lic_url, + spdx_lic_key, + lic_score, + lic_matched_text, + ) # FIXME: add docstring -def format_about_dict_for_csv_output(about_dictionary_list): - csv_formatted_list = [] +def format_about_dict_output(about_dictionary_list): + formatted_list = [] for element in about_dictionary_list: - row_list = OrderedDict() + row_list = dict() for key in element: if element[key]: if isinstance(element[key], list): - row_list[key] = u'\n'.join((element[key])) - elif key == u'about_resource': - row_list[key] = u'\n'.join((element[key].keys())) + row_list[key] = "\n".join((element[key])) + elif key == "about_resource": + row_list[key] = "\n".join((element[key].keys())) else: row_list[key] = element[key] - csv_formatted_list.append(row_list) - return csv_formatted_list + formatted_list.append(row_list) + return formatted_list # FIXME: add docstring def format_about_dict_for_json_output(about_dictionary_list): - licenses = ['license_key', 'license_name', 'license_file', 'license_url'] + licenses = ["license_key", "license_name", "license_file", "license_url"] json_formatted_list = [] for element in about_dictionary_list: - row_list = OrderedDict() + row_list = dict() # FIXME: aboid using parallel list... use an object instead license_key = [] license_name = [] @@ -517,16 +618,16 @@ def format_about_dict_for_json_output(about_dictionary_list): for key in element: if element[key]: # The 'about_resource' is an ordered dict - if key == 'about_resource': + if key == "about_resource": row_list[key] = list(element[key].keys())[0] elif key in licenses: - if key == 'license_key': + if key == "license_key": license_key = element[key] - elif key == 'license_name': + elif key == "license_name": license_name = element[key] - elif key == 'license_file': + elif key == "license_file": license_file = element[key] - elif key == 'license_url': + elif key == "license_url": license_url = element[key] else: row_list[key] = element[key] @@ -536,17 +637,17 @@ def format_about_dict_for_json_output(about_dictionary_list): if license_group: licenses_list = [] for lic_group in license_group: - lic_dict = OrderedDict() + lic_dict = dict() if lic_group[0]: - lic_dict['key'] = lic_group[0] + lic_dict["key"] = lic_group[0] if lic_group[1]: - lic_dict['name'] = lic_group[1] + lic_dict["name"] = lic_group[1] if lic_group[2]: - lic_dict['file'] = lic_group[2] + lic_dict["file"] = lic_group[2] if lic_group[3]: - lic_dict['url'] = lic_group[3] + lic_dict["url"] = lic_group[3] licenses_list.append(lic_dict) - row_list['licenses'] = licenses_list + row_list["licenses"] = licenses_list json_formatted_list.append(row_list) return json_formatted_list @@ -571,10 +672,194 @@ def filter_errors(errors, minimum_severity=WARNING): Return a list of unique `errors` Error object filtering errors that have a severity below `minimum_severity`. """ - return unique([e for e in errors if e.severity >= minimum_severity]) + return [e for e in errors if e.severity >= minimum_severity] + + +def create_dir(location): + """ + Create directory or directory tree at location, ensuring it is readable + and writeable. + """ + import stat + + if not os.path.exists(location): + os.makedirs(location) + os.chmod(location, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) + + +def get_temp_dir(sub_dir_path=None): + """ + Create a unique new temporary directory location. Create directories + identified by sub_dir_path if provided in this temporary directory. + Return the location for this unique directory joined with the + sub_dir_path if any. + """ + new_temp_dir = build_temp_dir() + + if sub_dir_path: + # create a sub directory hierarchy if requested + new_temp_dir = os.path.join(new_temp_dir, sub_dir_path) + create_dir(new_temp_dir) + return new_temp_dir + + +def build_temp_dir(prefix="attributecode-"): + """ + Create and return a new unique empty directory created in base_dir. + """ + import tempfile + + location = tempfile.mkdtemp(prefix=prefix) + create_dir(location) + return location + + +def get_file_text(file_name, reference): + """ + Return the file content from the license_file/notice_file field from the + given reference directory. + """ + error = "" + text = "" + file_path = os.path.join(reference, file_name) + if not os.path.exists(file_path): + msg = "The file " + file_path + " does not exist" + error = Error(CRITICAL, msg) + else: + with codecs.open(file_path, "rb", encoding="utf-8-sig", errors="replace") as txt: + # with io.open(file_path, encoding='utf-8') as txt: + text = txt.read() + return error, text + + +def convert_object_to_dict(about): + """ + Convert the list of field object + [Field(name='name', value=''), Field(name='version', value='')] + to a dictionary + """ + about_dict = {} + # Convert all the supported fields into a dictionary + fields_dict = getattr(about, "fields") + custom_fields_dict = getattr(about, "custom_fields") + supported_dict = {**fields_dict, **custom_fields_dict} + for field in supported_dict: + key = supported_dict[field].name + value = supported_dict[field].value + about_dict[key] = value + return about_dict + + +def load_scancode_json(location): + """ + Read the scancode JSON file at `location` and return a list of dictionaries. + """ + updated_results = [] + + with open(location) as json_file: + results = json.load(json_file) + results = results["files"] + # Rename the "path" to "about_resource" and update "name" from path value + for item in results: + updated_dict = {} + for key in item: + if key == "path": + updated_dict["about_resource"] = item[key] + updated_dict["name"] = os.path.basename(item[key]) + else: + updated_dict[key] = item[key] + updated_results.append(updated_dict) + return updated_results + + +def load_excel(location, worksheet=None): + """ + Read XLSX at `location`, return a list of ordered dictionaries, one + for each row. + """ + results = [] + errors = [] + import warnings + + # This is to prevent showing the: warn("Workbook contains no default style, apply openpyxl's default") + with warnings.catch_warnings(record=True): + input_bom = openpyxl.load_workbook(location) + sheetnames = input_bom.sheetnames + if worksheet: + if worksheet not in sheetnames: + import sys + + print("The input worksheet name does not exist. Exiting.") + sys.exit(1) + sheet_obj = input_bom[worksheet] + else: + sheet_obj = input_bom.active + print("Working on the " + sheet_obj.title + " worksheet.") + max_col = sheet_obj.max_column + + index = 1 + col_keys = [] + mapping_dict = {} + + while index <= max_col: + value = sheet_obj.cell(row=1, column=index).value + if value in col_keys: + msg = "Duplicated column name, " + str(value) + ", detected." + errors.append(Error(CRITICAL, msg)) + return errors, results + if value in mapping_dict: + value = mapping_dict[value] + col_keys.append(value) + index = index + 1 + + for row in sheet_obj.iter_rows(min_row=2, values_only=True): + row_dict = OrderedDict() + index = 0 + while index < max_col: + value = row[index] + if value: + row_dict[col_keys[index]] = value + else: + row_dict[col_keys[index]] = "" + index = index + 1 + results.append(row_dict) + return errors, results + + +def write_licenses(lic_dict, location): + import io + + loc = to_posix(location) + errors = [] + + if not posixpath.exists(loc): + os.makedirs(add_unc(loc)) + try: + for lic in lic_dict: + output_location = posixpath.join(loc, lic) + with open(output_location, "w", encoding="utf-8", errors="replace") as out: + out.write(lic_dict[lic]) + except Exception as e: + msg = str(e) + errors.append(Error(CRITICAL, msg)) + return errors + + +def strip_inventory_value(inventory): + """ + Strip the value of the dictionary and return the stripped dictionary to + a list. + """ + stripped_inventory = [] + for component in inventory: + comp_dict = {} + for key in component: + comp_dict[key] = str(component[key]).strip() + stripped_inventory.append(comp_dict) + return stripped_inventory """ Return True if a string s name is safe to use as an attribute name. """ -is_valid_name = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$').match +is_valid_name = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$").match diff --git a/templates/default_html.template b/templates/default_html.template deleted file mode 100644 index 5bffdf25..00000000 --- a/templates/default_html.template +++ /dev/null @@ -1,87 +0,0 @@ - - - - - Open Source Software Information - - - -

    OPEN SOURCE SOFTWARE INFORMATION

    -
    -

    Licenses, acknowledgments and required copyright notices for - open source components:

    -
    - -
    - {% for about_object in abouts %} -

    {{ about_object.name.value }}{% if about_object.version.value %} {{ about_object.version.value }}{% endif %}

    - {% endfor %} -
    - -
    - - - {% for about_object in abouts %} -
    -

    {{ about_object.name.value }} - {% if about_object.version.value %}{{ about_object.version.value }}{% endif %} -

    - {% if about_object.license_expression.value %} -

    This component is licensed under - {{ about_object.license_expression.value }} - {% endif %} - {% if about_object.copyright.value %} - 

    {{about_object.copyright.value}}
    - {% endif %} - {% if about_object.notice_file.value %} - {% for notice in about_object.notice_file.value %} - 
    {{ about_object.notice_file.value[notice] }}
    - {% endfor %} - {% endif %} - {% if about_object.license_key.value %} - {% for license_key in about_object.license_key.value %} - {% if license_key in common_licenses %} -

    Full text of - - {{ license_key }} - - is available at the end of this document.

    - {% endif %} - {% endfor %} - {% if about_object.license_file.value %} - {% for lic_file_name in about_object.license_file.value %} - {% if not license_file_name_and_key[lic_file_name] in common_licenses %} - 
    {{ about_object.license_file.value[lic_file_name] | e}}
    - {% endif %} - {% endfor %} - {% endif %} - {% else %} - {% if about_object.license_file.value %} - {% for lic_file_name in about_object.license_file.value %} - 
    {{ about_object.license_file.value[lic_file_name] | e}}
    - {% endfor %} - {% endif %} - {% endif %} -
    - {% endfor %} - -
    - -

    Common Licenses Used in This Product

    - - {% for key in license_key_and_context %} - {% if key in common_licenses %} -

    {{ key }}

    - 
    {{ license_key_and_context[key]|e }}
    - {% endif %} - {% endfor %} - -

    End

    - This file was generated with AboutCode Toolkit version: {{ tkversion }} on: {{ utcnow }} (UTC) - - - diff --git a/tests/test_api.py b/tests/test_api.py index 60764879..6597ffdf 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2017 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,13 +14,9 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - +import requests import unittest - -import mock +from unittest import mock from attributecode import api from attributecode import ERROR @@ -32,51 +28,65 @@ class FakeResponse(object): def __init__(self, response_content): self.response_content = response_content + self.text = response_content def read(self): return self.response_content class ApiTest(unittest.TestCase): - - @mock.patch.object(api, 'request_license_data') + @mock.patch.object(api, "request_license_data") def test_api_get_license_details_from_api(self, request_license_data): license_data = { - 'name': 'Apache License 2.0', - 'full_text': 'Apache License Version 2.0 ...', - 'key': 'apache-2.0', + "short_name": "Apache 2.0", + "full_text": "Apache License Version 2.0 ...", + "key": "apache-2.0", } errors = [] request_license_data.return_value = license_data, errors expected = ( - 'Apache License 2.0', - 'apache-2.0', - 'Apache License Version 2.0 ...', - []) + { + "short_name": "Apache 2.0", + "full_text": "Apache License Version 2.0 ...", + "key": "apache-2.0", + }, + [], + ) result = api.get_license_details_from_api( - api_url='api_url', api_key='api_key', license_key='license_key') + api_url="api_url", api_key="api_key", license_key="license_key" + ) assert expected == result - @mock.patch.object(api, 'urlopen') + @mock.patch.object(api, "get") def test_api_request_license_data_with_result(self, mock_data): response_content = ( b'{"count":1,"results":[{"name":"Apache 2.0","key":"apache-2.0","text":"Text"}]}' ) mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data( - api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') - expected = ( - {'name': 'Apache 2.0', 'key': 'apache-2.0', 'text': 'Text'}, - [] + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" ) + expected = ({"name": "Apache 2.0", "key": "apache-2.0", "text": "Text"}, []) assert expected == license_data - @mock.patch.object(api, 'urlopen') + @mock.patch.object(api, "get") def test_api_request_license_data_without_result(self, mock_data): response_content = b'{"count":0,"results":[]}' mock_data.return_value = FakeResponse(response_content) license_data = api.request_license_data( - api_url='http://fake.url/', api_key='api_key', license_key='apache-2.0') + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" + ) expected = ({}, [Error(ERROR, "Invalid 'license': apache-2.0")]) assert expected == license_data + + @mock.patch.object(api, "get") + def test_api_request_license_data_with_incorrect_url(self, mock_data): + # Some URL that is accessible but not a correct API URL + response_content = b"" + mock_data.return_value = FakeResponse(response_content) + license_data = api.request_license_data( + api_url="http://fake.url/", api_key="api_key", license_key="apache-2.0" + ) + expected = ({}, [Error(ERROR, "Invalid '--api_url'. License generation is skipped.")]) + assert expected == license_data diff --git a/tests/test_attrib.py b/tests/test_attrib.py index fd3b4403..da7c69e1 100644 --- a/tests/test_attrib.py +++ b/tests/test_attrib.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,10 +14,6 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import io import os import unittest @@ -25,46 +21,48 @@ from testing_utils import get_test_loc from testing_utils import get_temp_file +from attributecode import INFO from attributecode import attrib +from attributecode import gen from attributecode import model -class TemplateTest(unittest.TestCase): +class TemplateTest(unittest.TestCase): def test_check_template_simple_valid_returns_None(self): expected = None - assert expected == attrib.check_template('template_string') + assert expected == attrib.check_template("template_string") def test_check_template_complex_valid_returns_None(self): - template = ''' + template = """ {% for about in abouts -%} {{ about.name.value }}: {{ about.version.value }} {% for res in about.about_resource.value -%} resource: {{ res }} {% endfor -%} - {% endfor -%}''' + {% endfor -%}""" expected = None assert expected == attrib.check_template(template) def test_check_template_complex_invalid_returns_error(self): - template = ''' + template = """ {% for about in abouts -%} {{ about.name.value }}: {{ about.version.value }} {% for res in about.about_ressdsdsdsdsdsdource.value -%} resource: {{] res }} {% endfor -%} - {% endfor -%}''' + {% endfor -%}""" expected = (5, "unexpected ']'") assert expected == attrib.check_template(template) def test_check_template_invalid_return_error_lineno_and_message(self): expected = 1, "unexpected end of template, expected 'end of print statement'." - assert expected == attrib.check_template('{{template_string') + assert expected == attrib.check_template("{{template_string") def test_check_template_all_builtin_templates_are_valid(self): builtin_templates_dir = os.path.dirname(attrib.DEFAULT_TEMPLATE_FILE) for template in os.listdir(builtin_templates_dir): template_loc = os.path.join(builtin_templates_dir, template) - with io.open(template_loc, 'r', encoding='utf-8') as tmpl: + with open(template_loc, "r", encoding="utf-8", errors="replace") as tmpl: template = tmpl.read() try: assert None == attrib.check_template(template) @@ -73,61 +71,270 @@ def test_check_template_all_builtin_templates_are_valid(self): class GenerateTest(unittest.TestCase): - def test_generate_from_collected_inventory_wih_custom_temaplte(self): - test_file = get_test_loc('test_attrib/gen_simple/attrib.ABOUT') + test_file = get_test_loc("test_attrib/gen_simple/attrib.ABOUT") errors, abouts = model.collect_inventory(test_file) assert not errors - test_template = get_test_loc('test_attrib/gen_simple/test.template') + test_template = get_test_loc("test_attrib/gen_simple/test.template") with open(test_template) as tmpl: template = tmpl.read() - expected = ( - 'Apache HTTP Server: 2.4.3\n' - 'resource: httpd-2.4.3.tar.gz\n') + expected = "Apache HTTP Server: 2.4.3\nresource: httpd-2.4.3.tar.gz\n" - error, result = attrib.generate(abouts, template) + license_dict = {} + is_about_input = True + min_license_score = 0 + scancode = False + error, result = attrib.generate( + abouts, is_about_input, license_dict, scancode, min_license_score, template=template + ) assert expected == result assert not error def test_generate_with_default_template(self): - test_file = get_test_loc('test_attrib/gen_default_template/attrib.ABOUT') + test_file = get_test_loc("test_attrib/gen_default_template/attrib.ABOUT") errors, abouts = model.collect_inventory(test_file) assert not errors - error, result = attrib.generate_from_file(abouts) + license_dict = {} + is_about_input = True + min_license_score = 0 + scancode = False + + error, result = attrib.generate_from_file( + abouts, is_about_input, license_dict, scancode, min_license_score + ) assert not error expected_file = get_test_loc( - 'test_attrib/gen_default_template/expected_default_attrib.html') + "test_attrib/gen_default_template/expected_default_attrib.html" + ) with open(expected_file) as exp: expected = exp.read() # strip the timestamp: the timestamp is wrapped in italic block result = remove_timestamp(result) expected = remove_timestamp(expected) + # Ignore all white spaces and newline + result = result.replace("\n", "").replace(" ", "") + expected = expected.replace("\n", "").replace(" ", "") assert expected == result def test_lic_key_name_sync(self): - test_file = get_test_loc('test_attrib/gen_license_key_name_check/test.ABOUT') - expected = get_test_loc('test_attrib/gen_license_key_name_check/expected/expected.html') - template_loc = get_test_loc('test_attrib/gen_license_key_name_check/custom.template') + test_file = get_test_loc("test_attrib/gen_license_key_name_check/test.ABOUT") + expected = get_test_loc("test_attrib/gen_license_key_name_check/expected/expected.html") + template_loc = get_test_loc("test_attrib/gen_license_key_name_check/custom.template") output_file = get_temp_file() + license_dict = {} + is_about_input = True + errors, abouts = model.collect_inventory(test_file) - attrib.generate_and_save(abouts, output_file, template_loc) + attrib.generate_and_save( + abouts, is_about_input, license_dict, output_file, template_loc=template_loc + ) with open(output_file) as of: - f1 = '\n'.join(of.readlines(False)) + f1 = [line.strip() for line in of if line.strip()] with open(expected) as ef: - f2 = '\n'.join(ef.readlines(False)) + f2 = [line.strip() for line in ef if line.strip()] assert f1 == f2 + def test_scancode_input_min_score_0(self): + test_file = get_test_loc("test_attrib/scancode_input/sc-2-licenses.json") + errors, abouts = gen.load_inventory(test_file, scancode=True) + # Check if there is error's level > INFO + result = [(level, e) for level, e in errors if level > INFO] + assert result == [] + + is_about_input = False + scancode = True + + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) + errors, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) + assert not errors + + expected_file = get_test_loc("test_attrib/scancode_input/sc-min_score-0.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # For whatever reasons, the directly comparison between the result and the + # expected doesn't work well, it works after removed all the newline and spaces + # assert expected == result + # assert expected.splitlines(False) == result.splitlines(False) + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") + + def test_scancode_input_min_score_100(self): + test_file = get_test_loc("test_attrib/scancode_input/sc-2-licenses.json") + errors, abouts = gen.load_inventory(test_file, scancode=True) + # Check if there is error's level > INFO + result = [(level, e) for level, e in errors if level > INFO] + assert result == [] + + is_about_input = False + scancode = True + + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) + errors, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=100 + ) + assert not errors + + expected_file = get_test_loc("test_attrib/scancode_input/sc.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # For whatever reasons, the directly comparison between the result and the + # expected doesn't work well, it works after removed all the newline and spaces + # assert expected == result + # assert expected.splitlines(False) == result.splitlines(False) + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") + + def test_scancode_input_dup_lic(self): + test_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic.json") + errors, abouts = gen.load_inventory(test_file, scancode=True) + # Check if there is error's level > INFO + result = [(level, e) for level, e in errors if level > INFO] + assert result == [] + + is_about_input = False + scancode = True + + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) + errors, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) + assert not errors + + expected_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # For whatever reasons, the directly comparison between the result and the + # expected doesn't work well, it works after removed all the newline and spaces + # assert expected == result + # assert expected.splitlines(False) == result.splitlines(False) + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") + + def test_scancode_input_dup_lic_match(self): + test_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic-match.json") + errors, abouts = gen.load_inventory(test_file, scancode=True) + print("############################") + print(errors) + # Check if there is error's level > INFO + result = [(level, e) for level, e in errors if level > INFO] + assert result == [] + + is_about_input = False + scancode = True + + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) + errors, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) + assert not errors + + expected_file = get_test_loc("test_attrib/scancode_input/sc-dup-lic-match.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # For whatever reasons, the directly comparison between the result and the + # expected doesn't work well, it works after removed all the newline and spaces + # assert expected == result + # assert expected.splitlines(False) == result.splitlines(False) + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") + + def test_scancode_input_multi_lic(self): + test_file = get_test_loc("test_attrib/scancode_input/sc-multi-lic.json") + errors, abouts = gen.load_inventory(test_file, scancode=True) + # Check if there is error's level > INFO + result = [(level, e) for level, e in errors if level > INFO] + assert result == [] + + is_about_input = False + scancode = True + + lic_dict, _lic_errors = model.pre_process_and_fetch_license_dict(abouts) + errors, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) + assert not errors + + expected_file = get_test_loc("test_attrib/scancode_input/sc-multi-lic.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # For whatever reasons, the directly comparison between the result and the + # expected doesn't work well, it works after removed all the newline and spaces + # assert expected == result + # assert expected.splitlines(False) == result.splitlines(False) + assert expected.replace("\n", "").replace(" ", "").replace("\t", "") == result.replace( + "\n", "" + ).replace(" ", "").replace("\t", "") + + def test_generate_with_csv(self): + test_file = get_test_loc("test_attrib/default_template/simple_sample.csv") + errors, abouts = gen.load_inventory(test_file) + + lic_dict = { + "isc": [ + "ISC License", + "isc.LICENSE", + 'Permission to use, copy, modify, and/or distribute this software for any purpose\nwith or without fee is hereby granted, provided that the above copyright notice\nand this permission notice appear in all copies.\n\nTHE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH\nREGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND\nFITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,\nINDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS\nOF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER\nTORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF\nTHIS SOFTWARE.\n', + "https://scancode-licensedb.aboutcode.org/isc.LICENSE", + ] + } + is_about_input = False + scancode = False + + error, result = attrib.generate_from_file( + abouts, is_about_input, lic_dict, scancode, min_license_score=0 + ) + assert not error + + expected_file = get_test_loc("test_attrib/default_template/expect.html") + with open(expected_file) as exp: + expected = exp.read() + + # strip the timestamp: the timestamp is wrapped in italic block + result = remove_timestamp(result) + expected = remove_timestamp(expected) + # assert expected == result + assert expected.replace("\n", "").replace(" ", "") == result.replace("\n", "").replace( + " ", "" + ) + + def remove_timestamp(html_text): """ Return the `html_text` generated attribution stripped from timestamps: the timestamp is wrapped in italic block in the default template. """ - return '\n'.join(x for x in html_text.splitlines() if not '' in x) + return "\n".join(x for x in html_text.splitlines() if not "" in x) diff --git a/tests/test_cmd.py b/tests/test_cmd.py index 80ebad99..50119e12 100644 --- a/tests/test_cmd.py +++ b/tests/test_cmd.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,10 +14,6 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import io import unittest @@ -35,302 +31,274 @@ from testing_utils import get_temp_dir from testing_utils import get_temp_file - # NB: the test_report_errors* tests depend on py.test stdout/err capture capabilities + def test_report_errors(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] ec = cmd.report_errors(errors, quiet=False, verbose=True, log_file_loc=None) - assert 3 == ec + assert 6 == ec out, err = capsys.readouterr() expected_out = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4'] - assert '' == err + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", + ] + assert "" == err assert expected_out == out.splitlines(False) def test_report_errors_without_verbose(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] ec = cmd.report_errors(errors, quiet=False, verbose=False, log_file_loc=None) assert 3 == ec out, err = capsys.readouterr() expected_out = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'WARNING: msg4', + "Command completed with 3 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "WARNING: msg4", ] - assert '' == err + assert "" == err assert expected_out == out.splitlines(False) def test_report_errors_with_quiet_ignores_verbose_flag(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=True) - assert severe_errors_count == 3 + assert severe_errors_count == 6 out, err = capsys.readouterr() - assert '' == out - assert '' == err + assert "" == out + assert "" == err def test_report_errors_with_quiet_ignores_verbose_flag2(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=False) assert severe_errors_count == 3 out, err = capsys.readouterr() - assert '' == out - assert '' == err + assert "" == out + assert "" == err + def test_report_errors_with_verbose_flag(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] severe_errors_count = cmd.report_errors(errors, quiet=False, verbose=True) - assert severe_errors_count == 3 + assert severe_errors_count == 6 out, err = capsys.readouterr() expected_out = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4' + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", ] assert expected_out == out.splitlines(False) - assert '' == err + assert "" == err def test_report_errors_can_write_to_logfile(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), + Error(WARNING, "msg4"), ] result_file = get_temp_file() - _ec = cmd.report_errors(errors, quiet=False, verbose=True, - log_file_loc=result_file) - with io.open(result_file, 'r', encoding='utf-8') as rf: + _ec = cmd.report_errors(errors, quiet=False, verbose=True, log_file_loc=result_file) + with open(result_file, "r", encoding="utf-8", errors="replace") as rf: result = rf.read() expected = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4' + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", ] assert expected == result.splitlines(False) def test_report_errors_does_not_report_duplicate_errors(capsys): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), # dupes - Error(WARNING, 'msg4'), - Error(CRITICAL, 'msg1'), + Error(WARNING, "msg4"), + Error(CRITICAL, "msg1"), ] severe_errors_count = cmd.report_errors(errors, quiet=True, verbose=True) - assert severe_errors_count == 3 + assert severe_errors_count == 6 def test_get_error_messages(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] emsgs, ec = cmd.get_error_messages(errors) assert 3 == ec expected = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'WARNING: msg4', - ] - assert expected == emsgs - - -def test_get_error_messages_quiet(): - errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + "Command completed with 3 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "WARNING: msg4", ] - - emsgs, ec = cmd.get_error_messages(errors, quiet=True) - assert 3 == ec - expected = [] assert expected == emsgs def test_get_error_messages_verbose(): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] emsgs, ec = cmd.get_error_messages(errors, verbose=True) - assert 3 == ec + assert 6 == ec expected = [ - 'Command completed with 3 errors or warnings.', - 'CRITICAL: msg1', - 'ERROR: msg2', - 'INFO: msg3', - 'WARNING: msg4', - 'DEBUG: msg4', - 'NOTSET: msg4'] + "Command completed with 6 errors or warnings.", + "CRITICAL: msg1", + "ERROR: msg2", + "INFO: msg3", + "WARNING: msg4", + "DEBUG: msg4", + "NOTSET: msg4", + ] assert expected == emsgs class TestFilterError(unittest.TestCase): def test_filter_errors_default(self): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] expected = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(WARNING, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(WARNING, "msg4"), ] assert expected == cmd.filter_errors(errors) - def test_filter_errors_with_min(self): errors = [ - Error(CRITICAL, 'msg1'), - Error(ERROR, 'msg2'), - Error(INFO, 'msg3'), - Error(WARNING, 'msg4'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(CRITICAL, "msg1"), + Error(ERROR, "msg2"), + Error(INFO, "msg3"), + Error(WARNING, "msg4"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] expected = [ - Error(CRITICAL, 'msg1'), + Error(CRITICAL, "msg1"), ] assert expected == cmd.filter_errors(errors, CRITICAL) - def test_filter_errors_no_errors(self): errors = [ - Error(INFO, 'msg3'), - Error(DEBUG, 'msg4'), - Error(NOTSET, 'msg4'), + Error(INFO, "msg3"), + Error(DEBUG, "msg4"), + Error(NOTSET, "msg4"), ] assert [] == cmd.filter_errors(errors) - def test_filter_errors_none(self): assert [] == cmd.filter_errors([]) class TestParseKeyValues(unittest.TestCase): - def test_parse_key_values_empty(self): assert ({}, []) == cmd.parse_key_values([]) assert ({}, []) == cmd.parse_key_values(None) - def test_parse_key_values_simple(self): test = [ - 'key=value', - 'This=THat', - 'keY=bar', + "key=value", + "This=THat", + "keY=bar", ] - expected = { - 'key': 'bar', - 'this': 'THat' - } + expected = {"key": "bar", "this": "THat"} keyvals, errors = cmd.parse_key_values(test) assert expected == keyvals assert not errors - def test_parse_key_values_with_errors(self): - test = [ - 'key', - '=THat', - 'keY=', - 'FOO=bar' - ] + test = ["key", "=THat", "keY=", "FOO=bar"] expected = { - 'foo': 'bar', + "foo": "bar", } keyvals, errors = cmd.parse_key_values(test) assert expected == keyvals expected = [ 'missing in "=THat".', 'missing in "keY=".', - 'missing in "key".' + 'missing in "key".', ] assert expected == errors @@ -339,6 +307,7 @@ def test_parse_key_values_with_errors(self): # Run full cli command ############################################################################### + def check_about_stdout(options, expected_loc, regen=False): """ Run the about command with the `options` list of options. Assert that @@ -348,81 +317,82 @@ def check_about_stdout(options, expected_loc, regen=False): result = run_about_command_test_click(options) if regen: expected_file = get_test_loc(expected_loc, must_exists=False) - with open(expected_file, 'wb') as ef: - ef.write(result.output_bytes) + with open(expected_file, "w") as ef: + ef.write(result.output) expected_file = get_test_loc(expected_loc, must_exists=True) - with open(expected_file, 'rb') as ef: + with open(expected_file, "r") as ef: expected = ef.read() - assert expected.splitlines(False) == result.output_bytes.splitlines(False) + print(result.output) + assert expected.splitlines(False) == result.output.splitlines(False) def test_about_help_text(): - check_about_stdout(['--help'], 'test_cmd/help/about_help.txt', regen=False) + check_about_stdout(["--help"], "test_cmd/help/about_help.txt", regen=False) def test_about_inventory_help_text(): check_about_stdout( - ['inventory', '--help'], - 'test_cmd/help/about_inventory_help.txt', regen=False) + ["inventory", "--help"], "test_cmd/help/about_inventory_help.txt", regen=False + ) def test_about_gen_help_text(): + check_about_stdout(["gen", "--help"], "test_cmd/help/about_gen_help.txt", regen=False) + + +def test_about_gen_license_help_text(): check_about_stdout( - ['gen', '--help'], - 'test_cmd/help/about_gen_help.txt', regen=False) + ["gen-license", "--help"], "test_cmd/help/about_gen_license_help.txt", regen=False + ) def test_about_check_help_text(): - check_about_stdout( - ['check', '--help'], - 'test_cmd/help/about_check_help.txt', regen=False) + check_about_stdout(["check", "--help"], "test_cmd/help/about_check_help.txt", regen=False) def test_about_attrib_help_text(): - check_about_stdout( - ['attrib', '--help'], - 'test_cmd/help/about_attrib_help.txt', regen=False) + check_about_stdout(["attrib", "--help"], "test_cmd/help/about_attrib_help.txt", regen=False) def test_about_command_fails_with_an_unknown_subcommand(): test_dir = get_temp_dir() - result = run_about_command_test_click(['foo', test_dir], expected_rc=2) - assert b'Error: No such command "foo".' in result.output_bytes + result = run_about_command_test_click(["foo", test_dir], expected_rc=2) + assert "Error: No such command 'foo'." in result.output def test_about_inventory_command_can_run_minimally_without_error(): - test_dir = get_test_loc('test_cmd/repository-mini') + test_dir = get_test_loc("test_cmd/repository-mini") result = get_temp_file() - run_about_command_test_click(['inventory', test_dir, result]) + run_about_command_test_click(["inventory", test_dir, result]) def test_about_gen_command_can_run_minimally_without_error(): - test_inv = get_test_loc('test_cmd/geninventory.csv') + test_inv = get_test_loc("test_cmd/geninventory.csv") gen_dir = get_temp_dir() - run_about_command_test_click(['gen', test_inv, gen_dir]) + run_about_command_test_click(["gen", test_inv, gen_dir]) def test_about_attrib_command_can_run_minimally_without_error(): - test_dir = get_test_loc('test_cmd/repository-mini') + test_dir = get_test_loc("test_cmd/repository-mini") result = get_temp_file() - run_about_command_test_click(['attrib', test_dir, result]) + run_about_command_test_click(["attrib", test_dir, result]) def test_about_transform_command_can_run_minimally_without_error(): - test_file = get_test_loc('test_cmd/transform.csv') - result = get_temp_file('file_name.csv') - run_about_command_test_click(['transform', test_file, result]) + test_file = get_test_loc("test_cmd/transform.csv") + result = get_temp_file("file_name.csv") + run_about_command_test_click(["transform", test_file, result]) def test_about_transform_help_text(): check_about_stdout( - ['transform', '--help'], - 'test_cmd/help/about_transform_help.txt', regen=False) + ["transform", "--help"], "test_cmd/help/about_transform_help.txt", regen=False + ) def test_about_transform_expanded_help_text(): check_about_stdout( - ['transform', '--help-format'], - 'test_cmd/help/about_transform_config_help.txt', regen=False) + ["transform", "--help-format"], "test_cmd/help/about_transform_config_help.txt", regen=False + ) diff --git a/tests/test_gen.py b/tests/test_gen.py index 0ecbff01..266622d2 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,19 +14,15 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict import unittest from testing_utils import get_temp_dir from testing_utils import get_test_loc +from attributecode import CRITICAL from attributecode import ERROR from attributecode import INFO -from attributecode import CRITICAL +from attributecode import WARNING from attributecode import Error from attributecode import gen from unittest.case import skip @@ -34,55 +30,81 @@ class GenTest(unittest.TestCase): def test_check_duplicated_columns(self): - test_file = get_test_loc('test_gen/dup_keys.csv') - expected = [Error(ERROR, 'Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.')] + test_file = get_test_loc("test_gen/dup_keys.csv") + expected = [ + Error( + ERROR, + "Duplicated column name(s): copyright with copyright\nPlease correct the input and re-run.", + ) + ] result = gen.check_duplicated_columns(test_file) assert expected == result def test_check_duplicated_columns_handles_lower_upper_case(self): - test_file = get_test_loc('test_gen/dup_keys_with_diff_case.csv') - expected = [Error(ERROR, 'Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.')] + test_file = get_test_loc("test_gen/dup_keys_with_diff_case.csv") + expected = [ + Error( + ERROR, + "Duplicated column name(s): copyright with Copyright\nPlease correct the input and re-run.", + ) + ] result = gen.check_duplicated_columns(test_file) assert expected == result def test_check_duplicated_about_resource(self): - test_dict = [ - {'about_resource': '/test/test.c', 'version': '1.03', 'name': 'test.c'}, - {'about_resource': '/test/abc/', 'version': '1.0', 'name': 'abc'}, - {'about_resource': '/test/test.c', 'version': '1.04', 'name': 'test1.c'}] - expected = [ - Error(CRITICAL, - "The input has duplicated values in 'about_resource' field: /test/test.c")] - result = gen.check_duplicated_about_resource(test_dict) - assert expected == result + arp_list = ["/test/test.c", "test/test1.h"] + arp1 = "/test/test.c" + arp2 = "/test/tmp/test.c" + expected = Error( + CRITICAL, "The input has duplicated values in 'about_resource' field: " + arp1 + ) + result1 = gen.check_duplicated_about_resource(arp1, arp_list) + result2 = gen.check_duplicated_about_resource(arp2, arp_list) + assert result1 == expected + assert result2 == "" def test_check_newline_in_file_field(self): - test_dict = [ - {'about_resource': '/test/test.c', 'name': 'test.c', 'notice_file': 'NOTICE\nNOTICE2'}, - {'about_resource': '/test/abc/', 'version': '1.0', 'name': 'abc'}, - {'about_resource': '/test/test.c', 'version': '1.04', 'name': 'test1.c'}] + test_dict1 = { + "about_resource": "/test/test.c", + "name": "test.c", + "notice_file": "NOTICE\nNOTICE2", + } + test_dict2 = { + "about_resource": "/test/test.c", + "name": "test.c", + "notice_file": "NOTICE, NOTICE2", + } expected = [ - Error(CRITICAL, - "New line character detected in 'notice_file' for '/test/test.c' which is not supported." - "\nPlease use ',' to declare multiple files.")] - result = gen.check_newline_in_file_field(test_dict) - assert expected == result + Error( + CRITICAL, + "New line character detected in 'notice_file' for '/test/test.c' which is not supported." + "\nPlease use ',' to declare multiple files.", + ) + ] + result1 = gen.check_newline_in_file_field(test_dict1) + result2 = gen.check_newline_in_file_field(test_dict2) + assert result1 == expected + assert result2 == [] + + def test_check_about_resource_filename(self): + arp1 = "/test/t@est.c" + arp2 = "/test/t|est.c" + msg = "Invalid characters present in 'about_resource' field: " + arp2 + expected2 = Error(ERROR, msg) + result1 = gen.check_about_resource_filename(arp1) + result2 = gen.check_about_resource_filename(arp2) + assert result1 == "" + assert result2 == expected2 def test_load_inventory(self): - location = get_test_loc('test_gen/inv.csv') + location = get_test_loc("test_gen/inv.csv") base_dir = get_temp_dir() - errors, abouts = gen.load_inventory(location, base_dir) + errors, abouts = gen.load_inventory(location, base_dir=base_dir) - expected_errors = [ - Error(INFO, 'Field custom1 is a custom field.'), - Error(INFO, 'Field about_resource: Path') - ] - for exp, err in zip(expected_errors, errors): - assert exp.severity == err.severity - assert err.message.startswith(exp.message) + expected_num_errors = 29 + assert len(errors) == expected_num_errors - expected = ( -'''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -91,96 +113,197 @@ def test_load_inventory(self): custom1: | multi line -''' - ) +""" result = [a.dumps() for a in abouts] assert expected == result[0] - def test_load_inventory_with_errors(self): - location = get_test_loc('test_gen/inv4.csv') + def test_load_inventory_without_about_resource(self): + location = get_test_loc("test_gen/inv_no_about_resource.csv") base_dir = get_temp_dir() - errors, abouts = gen.load_inventory(location, base_dir) + from_attrib = False + errors, abouts = gen.load_inventory(location, base_dir=base_dir, from_attrib=from_attrib) + expected = """name: AboutCode +version: 0.11.0 +license_expression: apache-2.0 +licenses: + - key: apache-2.0 + name: apache-2.0 +""" + + assert errors == [] + result = [a.dumps() for a in abouts] + assert expected == result[0] + + def test_load_inventory_without_about_resource_from_attrib(self): + location = get_test_loc("test_gen/inv_no_about_resource.csv") + base_dir = get_temp_dir() + from_attrib = True + errors, abouts = gen.load_inventory(location, base_dir=base_dir, from_attrib=from_attrib) + + expected_num_errors = 0 + assert len(errors) == expected_num_errors + expected = """name: AboutCode +version: 0.11.0 +license_expression: apache-2.0 +licenses: + - key: apache-2.0 + name: apache-2.0 +""" + result = [a.dumps() for a in abouts] + assert expected == result[0] + + def test_load_inventory_with_errors(self): + location = get_test_loc("test_gen/inv4.csv") + base_dir = get_temp_dir() + errors, abouts = gen.load_inventory(location, base_dir=base_dir) expected_errors = [ - Error(CRITICAL, "Field name: 'confirmed copyright' contains illegal name characters: 0 to 9, a to z, A to Z and _."), - Error(INFO, 'Field resource is a custom field.'), - Error(INFO, 'Field test is a custom field.'), - Error(INFO, 'Field about_resource: Path') + Error( + WARNING, + "Field name: ['confirmed copyright'] contains illegal name characters (or empty spaces) and is ignored.", + ), + Error(INFO, "Field about_resource: Path"), + Error(INFO, "Field ['resource', 'test'] is a custom field."), ] - # assert [] == errors + for exp, err in zip(expected_errors, errors): assert exp.severity == err.severity assert err.message.startswith(exp.message) expected = ( - 'about_resource: .\n' - 'name: AboutCode\n' - 'version: 0.11.0\n' - 'description: |\n' - ' multi\n' - ' line\n' + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "description: |\n" + " multi\n" + " line\n" # 'confirmed copyright: Copyright (c) nexB, Inc.\n' - 'resource: this.ABOUT\n' - 'test: This is a test\n' + "resource: this.ABOUT\n" + "test: This is a test\n" ) result = [a.dumps() for a in abouts] assert expected == result[0] + def test_load_inventory_simple_xlsx(self): + location = get_test_loc("test_gen/load/simple_sample.xlsx") + base_dir = get_temp_dir() + errors, abouts = gen.load_inventory(location, base_dir=base_dir) + expected_errors = [] + result = [(level, e) for level, e in errors if level > INFO] + assert expected_errors == result + + assert abouts[0].name.value == "cryptohash-sha256" + assert abouts[1].name.value == "some_component" + + assert abouts[0].version.value == "v 0.11.100.1" + assert abouts[1].version.value == "v 0.0.1" + + assert abouts[0].license_expression.value == "bsd-new and mit" + assert abouts[1].license_expression.value == "mit" + + def test_load_scancode_json(self): + location = get_test_loc("test_gen/load/clean-text-0.3.0-lceupi.json") + inventory = gen.load_scancode_json(location) + + expected = { + "about_resource": "clean-text-0.3.0", + "type": "directory", + "name": "clean-text-0.3.0", + "base_name": "clean-text-0.3.0", + "extension": "", + "size": 0, + "date": None, + "sha1": None, + "md5": None, + "sha256": None, + "mime_type": None, + "file_type": None, + "programming_language": None, + "is_binary": False, + "is_text": False, + "is_archive": False, + "is_media": False, + "is_source": False, + "is_script": False, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 9, + "dirs_count": 1, + "size_count": 32826, + "scan_errors": [], + } + + # We will only check the first element in the inventory list + assert inventory[0] == expected + def test_generation_dir_endswith_space(self): - location = get_test_loc('test_gen/inventory/complex/about_file_path_dir_endswith_space.csv') + location = get_test_loc("test_gen/inventory/complex/about_file_path_dir_endswith_space.csv") base_dir = get_temp_dir() errors, _abouts = gen.generate(location, base_dir) - expected_errors_msg1 = 'contains directory name ends with spaces which is not allowed. Generation skipped.' - expected_errors_msg2 = 'Field about_resource' + expected_errors_msg1 = ( + "contains directory name ends with spaces which is not allowed. Generation skipped." + ) + expected_errors_msg2 = "Field about_resource" assert errors assert len(errors) == 2 - assert expected_errors_msg1 in errors[0].message or expected_errors_msg1 in errors[1].message - assert expected_errors_msg2 in errors[0].message or expected_errors_msg2 in errors[1].message + assert ( + expected_errors_msg1 in errors[0].message or expected_errors_msg1 in errors[1].message + ) + assert ( + expected_errors_msg2 in errors[0].message or expected_errors_msg2 in errors[1].message + ) def test_generation_with_no_about_resource(self): - location = get_test_loc('test_gen/inv2.csv') + location = get_test_loc("test_gen/inv2.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = OrderedDict([('.', None)]) + expected = dict([(".", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 def test_generation_with_no_about_resource_reference(self): - location = get_test_loc('test_gen/inv3.csv') + location = get_test_loc("test_gen/inv3.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = OrderedDict([('test.tar.gz', None)]) + expected = dict([("test.tar.gz", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 - msg = 'Field about_resource' + msg = "Field about_resource" assert msg in errors[0].message def test_generation_with_no_about_resource_reference_no_resource_validation(self): - location = get_test_loc('test_gen/inv3.csv') + location = get_test_loc("test_gen/inv3.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - expected = OrderedDict([('test.tar.gz', None)]) + expected = dict([("test.tar.gz", None)]) assert abouts[0].about_resource.value == expected assert len(errors) == 1 def test_generate(self): - location = get_test_loc('test_gen/inv.csv') + location = get_test_loc("test_gen/inv.csv") base_dir = get_temp_dir() errors, abouts = gen.generate(location, base_dir) - msg1 = 'Field custom1 is a custom field.' - msg2 = 'Field about_resource' + err_msg_list = [] + for severity, message in errors: + err_msg_list.append(message) + msg1 = "Field ['custom1'] is a custom field." - assert msg1 in errors[0].message - assert msg2 in errors[1].message + assert msg1 in err_msg_list result = [a.dumps() for a in abouts][0] - expected = ( -'''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -189,40 +312,238 @@ def test_generate(self): custom1: | multi line -''' - ) +""" assert expected == result - @skip('FIXME: this test is making a failed, live API call') + def test_generate(self): + location = get_test_loc("test_gen/inv.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + err_msg_list = [] + for severity, message in errors: + err_msg_list.append(message) + msg1 = "Field ['custom1'] is a custom field." + + assert msg1 in err_msg_list + + result = [a.dumps() for a in abouts][0] + expected = """about_resource: . +name: AboutCode +version: 0.11.0 +description: | + multi + line +custom1: | + multi + line +""" + assert expected == result + + def test_generate_multi_lic_issue_443(self): + location = get_test_loc("test_gen/multi_lic_issue_443/test.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + result = [a.dumps() for a in abouts][0] + expected = """about_resource: test +name: test +version: '1.5' +licenses: + - key: License1 + name: License1 + file: LIC1.LICENSE + - key: License2 + name: License2 + file: LIC2.LICENSE + - key: License3 + name: License3 + file: LIC3.LICENSE +""" + assert expected == result + + def test_generate_multi_lic_issue_444(self): + location = get_test_loc("test_gen/multi_lic_issue_444/test1.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + result = [a.dumps() for a in abouts][0] + expected = """about_resource: test.c +name: test.c +licenses: + - key: License1 + name: License1 + file: LIC1.LICENSE, LIC2.LICENSE +""" + assert expected == result + + def test_generate_license_key_with_custom_file_450_no_fetch(self): + location = get_test_loc("test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + result = [a.dumps() for a in abouts][0] + expected = """about_resource: test.c +name: test.c +license_expression: mit AND custom +licenses: + - key: mit + name: mit + - key: custom + name: custom + - file: custom.txt +""" + assert expected == result + + def test_generate_with_no_license_key_custom_lic_file(self): + location = get_test_loc( + "test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv" + ) + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + # The first row from the test file + a = abouts[0] + result1 = a.dumps() + + expected1 = """about_resource: test.c +name: test.c +licenses: + - file: custom.txt +""" + assert expected1 == result1 + + def test_generate_with_license_key_custom_lic_file(self): + location = get_test_loc("test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + # The first row from the test file + a = abouts[0] + result1 = a.dumps() + + expected1 = """about_resource: test.c +name: test.c +license_expression: custom +licenses: + - key: custom + name: custom + file: custom.txt +""" + assert expected1 == result1 + + def test_generate_license_key_with_custom_file_450_with_fetch_with_order(self): + location = get_test_loc("test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv") + base_dir = get_temp_dir() + + errors, abouts = gen.generate(location, base_dir) + + lic_dict = { + "mit": [ + "MIT License", + "mit.LICENSE", + "This component is released under MIT License.", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "mit", + ] + } + # The first row from the test file + a = abouts[0] + a.license_key.value.append("mit") + a.license_key.value.append("custom") + result1 = a.dumps(lic_dict) + # The second row from the test file + b = abouts[1] + b.license_key.value.append("custom") + b.license_key.value.append("mit") + result2 = b.dumps(lic_dict) + + expected1 = """about_resource: test.c +name: test.c +license_expression: mit AND custom +licenses: + - key: mit + name: MIT License + file: mit.LICENSE + url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit + spdx_license_key: mit + - key: custom + name: custom + file: custom.txt +""" + + expected2 = """about_resource: test.h +name: test.h +license_expression: custom AND mit +licenses: + - key: custom + name: custom + file: custom.txt + - key: mit + name: MIT License + file: mit.LICENSE + url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit + spdx_license_key: mit +""" + assert expected1 == result1 + assert expected2 == result2 + + @skip("FIXME: this test is making a failed, live API call") def test_generate_not_overwrite_original_license_file(self): - location = get_test_loc('test_gen/inv5.csv') + location = get_test_loc("test_gen/inv5.csv") base_dir = get_temp_dir() reference_dir = None - fetch_license = ['url', 'lic_key'] + fetch_license = ["url", "lic_key"] - _errors, abouts = gen.generate( - location, base_dir, reference_dir, fetch_license) + _errors, abouts = gen.generate(location, base_dir, reference_dir, fetch_license) - result = [a.dumps()for a in abouts][0] + result = [a.dumps() for a in abouts][0] expected = ( - 'about_resource: .\n' - 'name: AboutCode\n' - 'version: 0.11.0\n' - 'licenses:\n' - ' - file: this.LICENSE\n') + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "licenses:\n" + " - file: this.LICENSE\n" + ) + assert expected == result + + def test_generate_new_lic_fields_563(self): + location = get_test_loc("test_gen/inv7.csv") + base_dir = get_temp_dir() + + _errors, abouts = gen.generate(location, base_dir) + + result = [a.dumps() for a in abouts][0] + expected = """about_resource: test.c +name: test.c +license_expression: mit +declared_license_expression: isc +other_license_expression: public-domain +copyright: robot +licenses: + - key: mit + name: mit +""" assert expected == result def test_boolean_value_not_lost(self): - location = get_test_loc('test_gen/inv6.csv') + location = get_test_loc("test_gen/inv6.csv") base_dir = get_temp_dir() _errors, abouts = gen.generate(location, base_dir) in_mem_result = [a.dumps() for a in abouts][0] - expected = (u'about_resource: .\n' - u'name: AboutCode\n' - u'version: 0.11.0\n' - u'redistribute: yes\n' - u'attribute: yes\n' - u'modified: no\n') - assert expected == in_mem_result \ No newline at end of file + expected = ( + "about_resource: .\n" + "name: AboutCode\n" + "version: 0.11.0\n" + "redistribute: yes\n" + "attribute: yes\n" + "modified: no\n" + ) + assert expected == in_mem_result diff --git a/tests/test_model.py b/tests/test_model.py index db04c93d..b5176276 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,18 +14,14 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict import io import json +import os import posixpath import shutil import unittest +from unittest import mock -import mock import saneyaml from attributecode import CRITICAL @@ -34,22 +30,16 @@ from attributecode import WARNING from attributecode import Error from attributecode import model -from attributecode.util import add_unc +from attributecode.util import add_unc, norm, on_windows from attributecode.util import load_csv from attributecode.util import to_posix from attributecode.util import replace_tab_with_spaces from testing_utils import extract_test_loc +from testing_utils import get_temp_dir from testing_utils import get_temp_file from testing_utils import get_test_loc -try: - # Python 2 - unicode # NOQA -except NameError: # pragma: nocover - # Python 3 - unicode = str # NOQA - def check_csv(expected, result, regen=False, fix_cell_linesep=False): """ @@ -76,8 +66,8 @@ def fix_crlf(items): This is fixing this until we find can why """ for key, value in items: - if isinstance(value, unicode) and '\r\n' in value: - value = value.replace('\r\n', '\n') + if isinstance(value, str) and "\r\n" in value: + value = value.replace("\r\n", "\n") yield key, value @@ -86,9 +76,9 @@ def check_json(expected, result): Assert that the contents of two JSON files are equal. """ with open(expected) as e: - expected = json.load(e, object_pairs_hook=OrderedDict) + expected = json.load(e, object_pairs_hook=dict) with open(result) as r: - result = json.load(r, object_pairs_hook=OrderedDict) + result = json.load(r, object_pairs_hook=dict) assert expected == result @@ -103,7 +93,7 @@ def get_unicode_content(location): """ Read file at location and return a unicode string. """ - with io.open(location, encoding='utf-8') as doc: + with open(location, encoding="utf-8", errors="replace") as doc: return doc.read() @@ -116,6 +106,7 @@ def test_Field_init(self): model.BooleanField() model.PathField() model.FileTextField() + model.PackageUrlField() def test_empty_Field_has_no_content(self): field = model.Field() @@ -123,12 +114,12 @@ def test_empty_Field_has_no_content(self): def test_empty_Field_has_default_value(self): field = model.Field() - assert '' == field.value + assert "" == field.value def test_PathField_check_location(self): - test_file = 'license.LICENSE' - field = model.PathField(name='f', value=test_file, present=True) - base_dir = get_test_loc('test_model/base_dir') + test_file = "license.LICENSE" + field = model.PathField(name="f", value=test_file, present=True) + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) expected_errrors = [] @@ -139,60 +130,66 @@ def test_PathField_check_location(self): assert expected == result def test_PathField_check_missing_location(self): - test_file = 'does.not.exist' - field = model.PathField(name='f', value=test_file, present=True) - base_dir = get_test_loc('test_model/base_dir') + test_file = "does.not.exist" + field = model.PathField(name="f", value=test_file, present=True) + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) file_path = posixpath.join(base_dir, test_file) - err_msg = 'Field f: Path %s not found' % file_path + err_msg = "Field f: Path %s not found" % file_path - expected_errors = [ - Error(CRITICAL, err_msg)] + expected_errors = [Error(CRITICAL, err_msg)] assert expected_errors == errors result = field.value[test_file] assert None == result def test_TextField_loads_file(self): - field = model.FileTextField( - name='f', value='license.LICENSE', present=True) + field = model.FileTextField(name="f", value="license.LICENSE", present=True) - base_dir = get_test_loc('test_model/base_dir') + base_dir = get_test_loc("test_model/base_dir") errors = field.validate(base_dir=base_dir) assert [] == errors - expected = {'license.LICENSE': 'some license text'} + expected = {"license.LICENSE": "some license text"} assert expected == field.value + def test_PackageUrlField_is_valid_url(self): + assert model.PackageUrlField.is_valid_purl("pkg:pypi/saneyaml@0.1") + + def test_PackageUrlField_is_valid_url_no_version(self): + assert model.PackageUrlField.is_valid_purl("pkg:pypi/saneyaml") + def test_UrlField_is_valid_url(self): - assert model.UrlField.is_valid_url('http://www.google.com') + assert model.UrlField.is_valid_url("http://www.google.com") def test_UrlField_is_valid_url_not_starting_with_www(self): - assert model.UrlField.is_valid_url('https://nexb.com') - assert model.UrlField.is_valid_url('http://archive.apache.org/dist/httpcomponents/commons-httpclient/2.0/source/commons-httpclient-2.0-alpha2-src.tar.gz') - assert model.UrlField.is_valid_url('http://de.wikipedia.org/wiki/Elf (Begriffsklärung)') - assert model.UrlField.is_valid_url('http://nothing_here.com') + assert model.UrlField.is_valid_url("https://nexb.com") + assert model.UrlField.is_valid_url( + "http://archive.apache.org/dist/httpcomponents/commons-httpclient/2.0/source/commons-httpclient-2.0-alpha2-src.tar.gz" + ) + assert model.UrlField.is_valid_url("http://de.wikipedia.org/wiki/Elf (Begriffsklärung)") + assert model.UrlField.is_valid_url("http://nothing_here.com") def test_UrlField_is_valid_url_no_schemes(self): - assert not model.UrlField.is_valid_url('google.com') - assert not model.UrlField.is_valid_url('www.google.com') - assert not model.UrlField.is_valid_url('') + assert not model.UrlField.is_valid_url("google.com") + assert not model.UrlField.is_valid_url("www.google.com") + assert not model.UrlField.is_valid_url("") def test_UrlField_is_valid_url_not_ends_with_com(self): - assert model.UrlField.is_valid_url('http://www.google') + assert model.UrlField.is_valid_url("http://www.google") def test_UrlField_is_valid_url_ends_with_slash(self): - assert model.UrlField.is_valid_url('http://www.google.co.uk/') + assert model.UrlField.is_valid_url("http://www.google.co.uk/") def test_UrlField_is_valid_url_empty_URL(self): - assert not model.UrlField.is_valid_url('http:') + assert not model.UrlField.is_valid_url("http:") def check_validate(self, field_class, value, expected, expected_errors): """ Check field values after validation """ - field = field_class(name='s', value=value, present=True) + field = field_class(name="s", value=value, present=True) # check that validate can be applied multiple times without side effects for _ in range(2): errors = field.validate() @@ -201,193 +198,197 @@ def check_validate(self, field_class, value, expected, expected_errors): def test_StringField_validate_trailing_spaces_are_removed(self): field_class = model.StringField - value = 'trailin spaces ' - expected = 'trailin spaces' + value = "trailin spaces " + expected = "trailin spaces" self.check_validate(field_class, value, expected, expected_errors=[]) def test_ListField_contains_list_after_validate(self): - value = 'string' + value = "string" field_class = model.ListField expected = [value] self.check_validate(field_class, value, expected, expected_errors=[]) def test_ListField_contains_stripped_strings_after_validate(self): - value = '''first line - second line ''' + value = """first line + second line """ field_class = model.ListField - expected = ['first line', 'second line'] + expected = ["first line", "second line"] self.check_validate(field_class, value, expected, expected_errors=[]) def test_PathField_contains_stripped_strings_after_validate(self): - value = '''first line - second line ''' + value = """first line + second line """ field_class = model.ListField - expected = ['first line', 'second line'] + expected = ["first line", "second line"] self.check_validate(field_class, value, expected, expected_errors=[]) def test_PathField_contains_dict_after_validate(self): - value = 'string' + value = "string" field_class = model.PathField - expected = OrderedDict([('string', None)]) + expected = dict([("string", None)]) expected_errors = [ - Error(ERROR, 'Field s: Unable to verify path: string: No base directory provided') - ] + Error(ERROR, "Field s: Unable to verify path: string: No base directory provided") + ] self.check_validate(field_class, value, expected, expected_errors) def test_SingleLineField_has_errors_if_multiline(self): - value = '''line1 - line2''' + value = """line1 + line2""" field_class = model.SingleLineField expected = value - expected_errors = [Error(ERROR, 'Field s: Cannot span multiple lines: line1\n line2')] + expected_errors = [ + Error(ERROR, "Field s: Cannot span multiple lines: line1\n line2") + ] self.check_validate(field_class, value, expected, expected_errors) class YamlParseTest(unittest.TestCase): maxDiff = None + def test_saneyaml_load_can_parse_simple_fields(self): - test = get_test_content('test_model/parse/basic.about') + test = get_test_content("test_model/parse/basic.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), + ("single_line", "optional"), + ("other_field", "value"), ] assert expected == list(result.items()) def test_saneyaml_load_does_not_convert_to_crlf(self): - test = get_test_content('test_model/crlf/about.ABOUT') + test = get_test_content("test_model/crlf/about.ABOUT") result = saneyaml.load(test) expected = [ - (u'about_resource', u'.'), - (u'name', u'pytest'), - (u'description', u'first line\nsecond line\nthird line\n'), - (u'copyright', u'copyright') + ("about_resource", "."), + ("name", "pytest"), + ("description", "first line\nsecond line\nthird line\n"), + ("copyright", "copyright"), ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_continuations(self): - test = get_test_content('test_model/parse/continuation.about') + test = get_test_content("test_model/parse/continuation.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), - (u'multi_line', u'some value and more and yet more') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more and yet more"), ] assert expected == list(result.items()) def test_saneyaml_load_can_handle_multiline_texts_and_strips_text_fields(self): - test = get_test_content('test_model/parse/complex.about') + test = get_test_content("test_model/parse/complex.about") result = saneyaml.load(test) expected = [ - ('single_line', 'optional'), - ('other_field', 'value'), - ('multi_line', 'some value and more and yet more'), - ('yetanother', 'sdasd')] + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more and yet more"), + ("yetanother", "sdasd"), + ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_verbatim_text_unstripped(self): - test = get_test_content('test_model/parse/continuation_verbatim.about') + test = get_test_content("test_model/parse/continuation_verbatim.about") result = saneyaml.load(test) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'some value \n and more \n and yet more \n \n') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value \n and more \n and yet more \n \n"), ] assert expected == list(result.items()) def test_saneyaml_load_can_parse_verbatim_tab_text_unstripped(self): - test = get_test_content('test_model/parse/continuation_verbatim_with_tab.about') + test = get_test_content("test_model/parse/continuation_verbatim_with_tab.about") data = replace_tab_with_spaces(test) result = saneyaml.load(data) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'This is a long description\nwith tab.\n') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "This is a long description\nwith tab.\n"), ] assert expected == list(result.items()) def test_saneyaml_load_report_error_for_invalid_field_name(self): - test = get_test_content('test_model/parse/invalid_names.about') + test = get_test_content("test_model/parse/invalid_names.about") try: saneyaml.load(test) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass def test_saneyaml_dangling_text_is_not_an_invalid_continuation(self): - test = get_test_content('test_model/parse/invalid_continuation.about') + test = get_test_content("test_model/parse/invalid_continuation.about") result = saneyaml.load(test) expected = [ - (u'single_line', u'optional'), - (u'other_field', u'value'), - (u'multi_line', u'some value and more\ninvalid continuation2') + ("single_line", "optional"), + ("other_field", "value"), + ("multi_line", "some value and more\ninvalid continuation2"), ] assert expected == list(result.items()) def test_saneyaml_load_accepts_unicode_keys_and_values(self): - test = get_test_content('test_model/parse/non_ascii_field_name_value.about') + test = get_test_content("test_model/parse/non_ascii_field_name_value.about") result = saneyaml.load(test) expected = [ - ('name', 'name'), - ('about_resource', '.'), - ('owner', 'Matías Aguirre'), - (u'Matías', u'unicode field name') + ("name", "name"), + ("about_resource", "."), + ("owner", "Matías Aguirre"), + ("Matías", "unicode field name"), ] assert expected == list(result.items()) def test_saneyaml_load_accepts_blank_lines_and_spaces_in_field_names(self): - test = ''' + test = """ name: test space version: 0.7.0 about_resource: about.py field with spaces: This is a test case for field with spaces -''' +""" result = saneyaml.load(test) expected = [ - ('name', 'test space'), - ('version', '0.7.0'), - ('about_resource', 'about.py'), - (u'field with spaces', u'This is a test case for field with spaces'), + ("name", "test space"), + ("version", "0.7.0"), + ("about_resource", "about.py"), + ("field with spaces", "This is a test case for field with spaces"), ] assert expected == list(result.items()) def test_saneyaml_loads_blank_lines_and_lines_without_no_colon(self): - test = ''' + test = """ name: no colon test test version: 0.7.0 about_resource: about.py test with no colon -''' +""" try: saneyaml.load(test) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass -class AboutTest(unittest.TestCase): +class AboutTest(unittest.TestCase): def test_About_load_ignores_original_field_order_and_uses_standard_predefined_order(self): # fields in this file are not in the standard order - test_file = get_test_loc('test_model/parse/ordered_fields.ABOUT') + test_file = get_test_loc("test_model/parse/ordered_fields.ABOUT") a = model.About(test_file) assert [] == a.errors - expected = ['about_resource', 'name', 'version', 'download_url'] + expected = ["about_resource", "name", "version", "download_url"] result = [f.name for f in a.all_fields() if f.present] assert expected == result @@ -395,14 +396,16 @@ def test_About_duplicate_field_names_are_detected_with_different_case(self): # This test is failing because the YAML does not keep the order when # loads the test files. For instance, it treat the 'About_Resource' as the # first element and therefore the dup key is 'about_resource'. - test_file = get_test_loc('test_model/parse/dupe_field_name.ABOUT') + test_file = get_test_loc("test_model/parse/dupe_field_name.ABOUT") a = model.About(test_file) expected = [ - Error(WARNING, 'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"'), - Error(WARNING, 'Field Name is a duplicate. Original value: "old" replaced with: "new"') + Error( + WARNING, + 'Field About_Resource is a duplicate. Original value: "." replaced with: "new value"', + ), + Error(WARNING, 'Field Name is a duplicate. Original value: "old" replaced with: "new"'), ] - result = a.errors assert sorted(expected) == sorted(result) @@ -410,30 +413,33 @@ def test_About_duplicate_field_names_are_not_reported_if_same_value(self): # This test is failing because the YAML does not keep the order when # loads the test files. For instance, it treat the 'About_Resource' as the # first element and therefore the dup key is 'about_resource'. - test_file = get_test_loc('test_model/parse/dupe_field_name_no_new_value.ABOUT') + test_file = get_test_loc("test_model/parse/dupe_field_name_no_new_value.ABOUT") a = model.About(test_file) - expected = [ -] + expected = [] result = a.errors assert sorted(expected) == sorted(result) def check_About_hydrate(self, about, fields): - expected = set([ - 'name', - 'homepage_url', - 'download_url', - 'version', - 'copyright', - 'date', - 'license_spdx', - 'license_text_file', - 'notice_file', - 'about_resource']) + expected = set( + [ + "name", + "homepage_url", + "download_url", + "version", + "copyright", + "date", + "license_spdx", + "license_text_file", + "notice_file", + "about_resource", + ] + ) expected_errors = [ - Error(INFO, 'Field date is a custom field.'), - Error(INFO, 'Field license_spdx is a custom field.'), - Error(INFO, 'Field license_text_file is a custom field.')] + Error(INFO, "Custom Field: date"), + Error(INFO, "Custom Field: license_spdx"), + Error(INFO, "Custom Field: license_text_file"), + ] errors = about.hydrate(fields) @@ -443,157 +449,166 @@ def check_About_hydrate(self, about, fields): assert expected == result def test_About_hydrate_normalize_field_names_to_lowercase(self): - test_content = get_test_content('test_gen/parser_tests/upper_field_names.ABOUT') + test_content = get_test_content("test_gen/parser_tests/upper_field_names.ABOUT") fields = saneyaml.load(test_content).items() a = model.About() for _ in range(3): self.check_About_hydrate(a, fields) def test_About_with_existing_about_resource_has_no_error(self): - test_file = get_test_loc('test_gen/parser_tests/about_resource_field.ABOUT') + test_file = get_test_loc("test_gen/parser_tests/about_resource_field.ABOUT") a = model.About(test_file) assert [] == a.errors - result = a.about_resource.value['about_resource.c'] + result = a.about_resource.value["about_resource.c"] # this means we have a location self.assertNotEqual([], result) - def test_About_has_errors_when_about_resource_is_missing(self): - test_file = get_test_loc('test_gen/parser_tests/.ABOUT') + def test_About_loads_ignored_resources_field(self): + # fields in this file are not in the standard order + test_file = get_test_loc("test_model/parse/with_ignored_resources.ABOUT") a = model.About(test_file) - expected = [Error(CRITICAL, 'Field about_resource is required')] - result = a.errors + # assert [] == a.errors + + expected = ["about_resource", "ignored_resources", "name"] + result = [f.name for f in a.all_fields() if f.present] assert expected == result + def test_About_has_no_errors_when_about_resource_is_missing(self): + test_file = get_test_loc("test_gen/parser_tests/.ABOUT") + a = model.About(test_file) + assert a.errors == [] + def test_About_has_errors_when_about_resource_does_not_exist(self): - test_file = get_test_loc('test_gen/parser_tests/missing_about_ref.ABOUT') - file_path = posixpath.join(posixpath.dirname(test_file), 'about_file_missing.c') + test_file = get_test_loc("test_gen/parser_tests/missing_about_ref.ABOUT") + file_path = posixpath.join(posixpath.dirname(test_file), "about_file_missing.c") a = model.About(test_file) - err_msg = 'Field about_resource: Path %s not found' % file_path + err_msg = "Field about_resource: Path %s not found" % file_path expected = [Error(INFO, err_msg)] result = a.errors assert expected == result def test_About_has_errors_when_missing_required_fields_are_missing(self): - test_file = get_test_loc('test_model/parse/missing_required.ABOUT') + test_file = get_test_loc("test_model/parse/missing_required.ABOUT") a = model.About(test_file) expected = [ - Error(CRITICAL, 'Field about_resource is required'), - Error(CRITICAL, 'Field name is required'), + Error(CRITICAL, "Field name is required"), ] result = a.errors assert expected == result def test_About_has_errors_when_required_fields_are_empty(self): - test_file = get_test_loc('test_model/parse/empty_required.ABOUT') + test_file = get_test_loc("test_model/parse/empty_required.ABOUT") a = model.About(test_file) expected = [ - Error(CRITICAL, 'Field about_resource is required and empty'), - Error(CRITICAL, 'Field name is required and empty'), + Error(CRITICAL, "Field name is required and empty"), ] result = a.errors assert expected == result def test_About_has_errors_with_empty_notice_file_field(self): - test_file = get_test_loc('test_model/parse/empty_notice_field.about') + test_file = get_test_loc("test_model/parse/empty_notice_field.about") a = model.About(test_file) - expected = [ - Error(INFO, 'Field notice_file is present but empty.')] + expected = [Error(INFO, "Field notice_file is present but empty.")] result = a.errors assert expected == result def test_About_custom_fields_are_never_ignored(self): - test_file = get_test_loc('test_model/custom_fields/custom_fields.about') + test_file = get_test_loc("test_model/custom_fields/custom_fields.about") a = model.About(test_file) result = [(n, f.value) for n, f in a.custom_fields.items()] expected = [ - (u'single_line', u'README STUFF'), - (u'multi_line', u'line1\nline2'), - (u'other', u'sasasas'), - (u'empty', u'') + ("single_line", "README STUFF"), + ("multi_line", "line1\nline2"), + ("other", "sasasas"), + ("empty", ""), ] assert expected == result def test_About_custom_fields_are_not_ignored_and_order_is_preserved(self): - test_file = get_test_loc('test_model/custom_fields/custom_fields.about') + test_file = get_test_loc("test_model/custom_fields/custom_fields.about") a = model.About(test_file) result = [(n, f.value) for n, f in a.custom_fields.items()] expected = [ - (u'single_line', u'README STUFF'), - (u'multi_line', u'line1\nline2'), - (u'other', u'sasasas'), - (u'empty', u'') + ("single_line", "README STUFF"), + ("multi_line", "line1\nline2"), + ("other", "sasasas"), + ("empty", ""), ] assert sorted(expected) == sorted(result) def test_About_has_errors_for_illegal_custom_field_name(self): - test_file = get_test_loc('test_model/parse/illegal_custom_field.about') + test_file = get_test_loc("test_model/parse/illegal_custom_field.about") a = model.About(test_file) expected_errors = [ - Error(INFO, 'Field hydrate is a custom field.'), - Error(CRITICAL, "Internal error with custom field: 'hydrate': 'illegal name'.") + Error(INFO, "Custom Field: hydrate"), + Error(CRITICAL, "Internal error with custom field: 'hydrate': 'illegal name'."), ] + assert expected_errors == a.errors - assert not hasattr(getattr(a, 'hydrate'), 'value') + assert not hasattr(getattr(a, "hydrate"), "value") field = list(a.custom_fields.values())[0] - assert 'hydrate' == field.name - assert 'illegal name' == field.value + assert "hydrate" == field.name + assert "illegal name" == field.value def test_About_file_fields_are_empty_if_present_and_path_missing(self): - test_file = get_test_loc('test_model/parse/missing_notice_license_files.ABOUT') + test_file = get_test_loc("test_model/parse/missing_notice_license_files.ABOUT") a = model.About(test_file) - file_path1 = posixpath.join(posixpath.dirname(test_file), 'test.LICENSE') - file_path2 = posixpath.join(posixpath.dirname(test_file), 'test.NOTICE') + file_path1 = posixpath.join(posixpath.dirname(test_file), "test.LICENSE") + file_path2 = posixpath.join(posixpath.dirname(test_file), "test.NOTICE") - err_msg1 = Error(CRITICAL, 'Field license_file: Path %s not found' % file_path1) - err_msg2 = Error(CRITICAL, 'Field notice_file: Path %s not found' % file_path2) + err_msg1 = Error(CRITICAL, "Field license_file: Path %s not found" % file_path1) + err_msg2 = Error(CRITICAL, "Field notice_file: Path %s not found" % file_path2) expected_errors = [err_msg1, err_msg2] assert expected_errors == a.errors - assert {'test.LICENSE': None} == a.license_file.value - assert {'test.NOTICE': None} == a.notice_file.value + assert {"test.LICENSE": None} == a.license_file.value + assert {"test.NOTICE": None} == a.notice_file.value def test_About_notice_and_license_text_are_loaded_from_file(self): - test_file = get_test_loc('test_model/parse/license_file_notice_file.ABOUT') + test_file = get_test_loc("test_model/parse/license_file_notice_file.ABOUT") a = model.About(test_file) - expected = '''Tester holds the copyright for test component. Tester relinquishes copyright of + expected = """Tester holds the copyright for test component. Tester relinquishes copyright of this software and releases the component to Public Domain. -* Email Test@tester.com for any questions''' +* Email Test@tester.com for any questions""" - result = a.license_file.value['license_text.LICENSE'] + result = a.license_file.value["license_text.LICENSE"] assert expected == result - expected = '''Test component is released to Public Domain.''' - result = a.notice_file.value['notice_text.NOTICE'] + expected = """Test component is released to Public Domain.""" + result = a.notice_file.value["notice_text.NOTICE"] assert expected == result def test_About_license_and_notice_text_are_empty_if_field_missing(self): - test_file = get_test_loc('test_model/parse/no_file_fields.ABOUT') + test_file = get_test_loc("test_model/parse/no_file_fields.ABOUT") a = model.About(test_file) assert [] == a.errors assert {} == a.license_file.value assert {} == a.notice_file.value def test_About_rejects_non_ascii_names_and_accepts_unicode_values(self): - test_file = get_test_loc('test_model/parse/non_ascii_field_name_value.about') + test_file = get_test_loc("test_model/parse/non_ascii_field_name_value.about") a = model.About(test_file) expected = [ - Error(CRITICAL, "Field name: 'mat\xedas' contains illegal name characters: 0 to 9, a to z, A to Z and _.") + Error( + WARNING, + "Field name: ['mat\xedas'] contains illegal name characters (or empty spaces) and is ignored.", + ) ] assert expected == a.errors def test_About_invalid_boolean_value(self): - test_file = get_test_loc('test_model/parse/invalid_boolean.about') + test_file = get_test_loc("test_model/parse/invalid_boolean.about") a = model.About(test_file) expected_msg = "Field modified: Invalid flag value: 'blah'" assert expected_msg in a.errors[0].message def test_About_boolean_value(self): - test_file = get_test_loc('test_model/parse/boolean_data.about') + test_file = get_test_loc("test_model/parse/boolean_data.about") a = model.About(test_file) expected_msg = "Field track_changes is present but empty." assert expected_msg in a.errors[0].message @@ -613,77 +628,172 @@ def test_About_boolean_value(self): assert a.redistribute.value is True assert a.track_changes.value is None + def test_About_boolean_numberic_value(self): + test_file = get_test_loc("test_model/parse/boolean_numeric_data.about") + a = model.About(test_file) + expected_msg = "Field track_changes is present but empty." + assert expected_msg in a.errors[0].message + # Context of the test file + """ + about_resource: . + name: boolean_data + attribute: 3 + modified: true + internal_use_only: no + redistribute: yes + track_changes: + """ + assert a.attribute.value == "3" + assert a.modified.value is True + assert a.internal_use_only.value is False + assert a.redistribute.value is True + assert a.track_changes.value is None + + def test_About_boolean_character_value(self): + test_file = get_test_loc("test_model/parse/boolean_chara_data.about") + a = model.About(test_file) + # Context of the test file + """ + about_resource: . + name: data + attribute: 11 + """ + assert a.attribute.value == "11" + assert len(a.errors) == 0 + + def test_About_boolean_more_than_2_character_value(self): + test_file = get_test_loc("test_model/parse/boolean_more_than_2_chara_data.about") + a = model.About(test_file) + expected_msg = "Path: None - Field attribute: Invalid value: 'abc' is not one of: ('yes', 'y', 'true', 'x', 'no', 'n', 'false') and it is not a 1 or 2 character value." + assert expected_msg in a.errors[0].message + # Context of the test file + """ + about_resource: . + name: test + attribute: abc + """ + assert a.attribute.value is None + def test_About_contains_about_file_path(self): - test_file = get_test_loc('test_model/serialize/about.ABOUT') + test_file = get_test_loc("test_model/serialize/about.ABOUT") # TODO: I am not sure this override of the about_file_path makes sense - a = model.About(test_file, about_file_path='complete/about.ABOUT') + a = model.About(test_file, about_file_path="complete/about.ABOUT") assert [] == a.errors - expected = 'complete/about.ABOUT' + expected = "complete/about.ABOUT" result = a.about_file_path assert expected == result def test_About_equals(self): - test_file = get_test_loc('test_model/equal/complete/about.ABOUT') - a = model.About(test_file, about_file_path='complete/about.ABOUT') - b = model.About(test_file, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/equal/complete/about.ABOUT") + a = model.About(test_file, about_file_path="complete/about.ABOUT") + b = model.About(test_file, about_file_path="complete/about.ABOUT") assert a == b def test_About_are_not_equal_with_small_text_differences(self): - test_file = get_test_loc('test_model/equal/complete2/about.ABOUT') - a = model.About(test_file, about_file_path='complete2/about.ABOUT') - test_file2 = get_test_loc('test_model/equal/complete/about.ABOUT') - b = model.About(test_file2, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/equal/complete2/about.ABOUT") + a = model.About(test_file, about_file_path="complete2/about.ABOUT") + test_file2 = get_test_loc("test_model/equal/complete/about.ABOUT") + b = model.About(test_file2, about_file_path="complete/about.ABOUT") assert a.dumps() != b.dumps() assert a != b def test_get_field_names_only_returns_non_empties(self): a = model.About() - a.custom_fields['f'] = model.StringField( - name='f', value='1', present=True) + a.custom_fields["f"] = model.StringField(name="f", value="1", present=True) b = model.About() - b.custom_fields['g'] = model.StringField( - name='g', value='1', present=True) + b.custom_fields["g"] = model.StringField(name="g", value="1", present=True) abouts = [a, b] # ensure all fields (including custom fields) and # about_resource are collected in the correct order - expected = [ - model.About.ABOUT_RESOURCE_ATTR, 'name', 'f', 'g' - ] + expected = ["name", "f", "g"] result = model.get_field_names(abouts) assert expected == result def test_get_field_names_does_not_return_duplicates_custom_fields(self): a = model.About() - a.custom_fields['f'] = model.StringField(name='f', value='1', - present=True) - a.custom_fields['cf'] = model.StringField(name='cf', value='1', - present=True) + a.custom_fields["f"] = model.StringField(name="f", value="1", present=True) + a.custom_fields["cf"] = model.StringField(name="cf", value="1", present=True) b = model.About() - b.custom_fields['g'] = model.StringField(name='g', value='1', - present=True) - b.custom_fields['cf'] = model.StringField(name='cf', value='2', - present=True) + b.custom_fields["g"] = model.StringField(name="g", value="1", present=True) + b.custom_fields["cf"] = model.StringField(name="cf", value="2", present=True) abouts = [a, b] # ensure all fields (including custom fields) and # about_resource are collected in the correct order expected = [ - 'about_resource', - 'name', - 'cf', - 'f', - 'g', - ] + "name", + "cf", + "f", + "g", + ] result = model.get_field_names(abouts) assert expected == result + def test_comma_in_license(self): + test_file = get_test_loc("test_model/special_char/about.ABOUT") + a = model.About(test_file) + expected = Error(ERROR, "The following character(s) cannot be in the license_key: [',']") + assert a.errors[0] == expected + + def test_load_dict_issue_433(self): + package_data = { + "about_resource": "package1.zip", + "name": "package", + "version": "1.0", + "copyright": "copyright on package", + "license_expression": "license1 AND license2", + "notice_file": "package1.zip.NOTICE", + "licenses": [ + { + "key": "license1", + "name": "License1", + "file": "license1.LICENSE", + "url": "some_url", + "spdx_license_key": "key", + }, + { + "key": "license2", + "name": "License2", + "file": "license2.LICENSE", + "url": "some_url", + "spdx_license_key": "key", + }, + ], + } + about = model.About() + about.load_dict(package_data, base_dir="") + as_dict = about.as_dict() + expected = """about_resource: package1.zip +name: package +version: '1.0' +license_expression: license1 AND license2 +copyright: copyright on package +notice_file: package1.zip.NOTICE +licenses: + - key: license1 + name: License1 + file: license1.LICENSE + url: some_url + spdx_license_key: key + - key: license2 + name: License2 + file: license2.LICENSE + url: some_url + spdx_license_key: key +""" + lic_dict = { + "license1": ["License1", "license1.LICENSE", "", "some_url", "key"], + "license2": ["License2", "license2.LICENSE", "", "some_url", "key"], + } + assert about.dumps(lic_dict) == expected + class SerializationTest(unittest.TestCase): def test_About_dumps(self): - test_file = get_test_loc('test_model/dumps/about.ABOUT') + test_file = get_test_loc("test_model/dumps/about.ABOUT") a = model.About(test_file) assert [] == a.errors - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 description: | @@ -700,39 +810,40 @@ def test_About_dumps(self): vcs_repository: https://github.com/dejacode/about-code-tool.git licenses: - key: apache-2.0 + name: Apache 2.0 file: apache-2.0.LICENSE -''' +""" result = a.dumps() assert expected == result def test_About_dumps_does_all_non_empty_present_fields(self): - test_file = get_test_loc('test_model/parse/complete2/about.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about.ABOUT") a = model.About(test_file) expected_error = [ - Error(INFO, 'Field custom1 is a custom field.'), - Error(INFO, 'Field custom2 is a custom field.'), - Error(INFO, 'Field custom2 is present but empty.') + Error(INFO, "Custom Field: custom1"), + Error(INFO, "Custom Field: custom2"), + Error(INFO, "Field custom2 is present but empty."), ] assert sorted(expected_error) == sorted(a.errors) - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 custom1: | multi line -''' +""" result = a.dumps() assert expected == result def test_About_dumps_with_different_boolean_value(self): - test_file = get_test_loc('test_model/parse/complete2/about2.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about2.ABOUT") a = model.About(test_file) expected_error_msg = "Field track_changes: Invalid flag value: 'blah' is not one of" assert len(a.errors) == 1 assert expected_error_msg in a.errors[0].message - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 @@ -740,46 +851,46 @@ def test_About_dumps_with_different_boolean_value(self): redistribute: no attribute: yes modified: yes -''' +""" result = a.dumps() assert set(expected) == set(result) def test_About_dumps_all_non_empty_fields(self): - test_file = get_test_loc('test_model/parse/complete2/about.ABOUT') + test_file = get_test_loc("test_model/parse/complete2/about.ABOUT") a = model.About(test_file) expected_error = [ - Error(INFO, 'Field custom1 is a custom field.'), - Error(INFO, 'Field custom2 is a custom field.'), - Error(INFO, 'Field custom2 is present but empty.') + Error(INFO, "Custom Field: custom1"), + Error(INFO, "Custom Field: custom2"), + Error(INFO, "Field custom2 is present but empty."), ] assert sorted(expected_error) == sorted(a.errors) - expected = '''about_resource: . + expected = """about_resource: . name: AboutCode version: 0.11.0 custom1: | multi line -''' +""" result = a.dumps() assert expected == result def test_About_as_dict_contains_special_paths(self): - test_file = get_test_loc('test_model/special/about.ABOUT') - a = model.About(test_file, about_file_path='complete/about.ABOUT') + test_file = get_test_loc("test_model/special/about.ABOUT") + a = model.About(test_file, about_file_path="complete/about.ABOUT") expected_errors = [] assert expected_errors == a.errors as_dict = a.as_dict() - expected = 'complete/about.ABOUT' + expected = "complete/about.ABOUT" result = as_dict[model.About.ABOUT_FILE_PATH_ATTR] assert expected == result def test_load_dump_is_idempotent(self): - test_file = get_test_loc('test_model/this.ABOUT') + test_file = get_test_loc("test_model/this.ABOUT") a = model.About() a.load(test_file) - dumped_file = get_temp_file('that.ABOUT') + dumped_file = get_temp_file("that.ABOUT") a.dump(dumped_file) expected = get_unicode_content(test_file).splitlines() @@ -787,60 +898,62 @@ def test_load_dump_is_idempotent(self): # Ignore comment and empty line filtered_result = [] for line in result: - if not line.startswith('#') and not line == '': + if not line.startswith("#") and not line == "": filtered_result.append(line) assert expected == filtered_result def test_load_can_load_unicode(self): - test_file = get_test_loc('test_model/unicode/nose-selecttests.ABOUT') + test_file = get_test_loc("test_model/unicode/nose-selecttests.ABOUT") a = model.About() a.load(test_file) - file_path = posixpath.join(posixpath.dirname(test_file), 'nose-selecttests-0.3.zip') - err_msg = 'Field about_resource: Path %s not found' % file_path + file_path = posixpath.join(posixpath.dirname(test_file), "nose-selecttests-0.3.zip") + err_msg = "Field about_resource: Path %s not found" % file_path errors = [ - Error(INFO, 'Field dje_license is a custom field.'), - Error(INFO, 'Field license_text_file is a custom field.'), - Error(INFO, 'Field scm_tool is a custom field.'), - Error(INFO, 'Field scm_repository is a custom field.'), - Error(INFO, 'Field test is a custom field.'), - Error(INFO, err_msg)] + Error(INFO, "Custom Field: dje_license"), + Error(INFO, "Custom Field: license_text_file"), + Error(INFO, "Custom Field: scm_tool"), + Error(INFO, "Custom Field: scm_repository"), + Error(INFO, "Custom Field: test"), + Error(INFO, err_msg), + ] assert errors == a.errors - assert 'Copyright (c) 2012, Domen Kožar' == a.copyright.value + assert "Copyright (c) 2012, Domen Kožar" == a.copyright.value - def test_load_has_errors_for_non_unicode(self): - test_file = get_test_loc('test_model/unicode/not-unicode.ABOUT') + def test_load_non_unicode(self): + test_file = get_test_loc("test_model/unicode/not-unicode.ABOUT") a = model.About() a.load(test_file) err = a.errors[0] assert CRITICAL == err.severity - assert 'Cannot load invalid ABOUT file' in err.message - assert 'UnicodeDecodeError' in err.message + assert "Cannot load invalid ABOUT file" in err.message def test_as_dict_load_dict_ignores_empties(self): test = { - 'about_resource': '.', - 'author': '', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'custom_empty': '', - 'description': 'AboutCode is a tool\nfor files.', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.'} + "about_resource": ".", + "author": "", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "custom_empty": "", + "description": "AboutCode is a tool\nfor files.", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + } expected = { - 'about_file_path': None, - 'about_resource': OrderedDict([('.', None)]), - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'description': 'AboutCode is a tool\nfor files.', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.'} + "about_file_path": None, + "about_resource": dict([(".", None)]), + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "description": "AboutCode is a tool\nfor files.", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + } a = model.About() - base_dir = 'some_dir' + base_dir = "some_dir" a.load_dict(test, base_dir) as_dict = a.as_dict() # FIXME: why converting back to dict? @@ -848,212 +961,257 @@ def test_as_dict_load_dict_ignores_empties(self): def test_load_dict_as_dict_is_idempotent_ignoring_special(self): test = { - 'about_resource': ['.'], - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0'} + "about_resource": ["."], + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", + } a = model.About() - base_dir = 'some_dir' + base_dir = "some_dir" a.load_dict(test, base_dir) as_dict = a.as_dict() expected = { - 'about_file_path': None, - 'about_resource': OrderedDict([('.', None)]), - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0'} + "about_file_path": None, + "about_resource": dict([(".", None)]), + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", + } assert expected == dict(as_dict) def test_about_model_class_from_dict_constructor(self): about_data = { - 'about_resource': ['.'], - 'attribute': 'yes', - 'author': 'Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'description': 'AboutCode is a tool to process ABOUT files. An ABOUT file is a file.', - 'homepage_url': 'http://dejacode.org', - 'license_expression': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.', - 'vcs_repository': 'https://github.com/dejacode/about-code-tool.git', - 'vcs_tool': 'git', - 'version': '0.11.0', + "about_resource": ["."], + "attribute": "yes", + "author": "Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "description": "AboutCode is a tool to process ABOUT files. An ABOUT file is a file.", + "homepage_url": "http://dejacode.org", + "license_expression": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", + "vcs_repository": "https://github.com/dejacode/about-code-tool.git", + "vcs_tool": "git", + "version": "0.11.0", } about = model.About.from_dict(about_data) assert isinstance(about, model.About) - about_data.update({ - 'about_file_path': None, - 'about_resource': OrderedDict([('.', None)]), - }) + about_data.update( + { + "about_file_path": None, + "about_resource": dict([(".", None)]), + } + ) assert about_data == about.as_dict() def test_write_output_csv(self): - path = 'test_model/this.ABOUT' + path = "test_model/this.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='csv') + model.write_output([abouts], result, format="csv") - expected = get_test_loc('test_model/expected.csv') + expected = get_test_loc("test_model/expected.csv") check_csv(expected, result) def test_write_output_csv_with_multiple_files(self): - path = 'test_model/multiple_files.ABOUT' + path = "test_model/multiple_files.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='csv') + model.write_output([abouts], result, format="csv") - expected = get_test_loc('test_model/multiple_files_expected.csv') + expected = get_test_loc("test_model/multiple_files_expected.csv") check_csv(expected, result) def test_write_output_json(self): - path = 'test_model/this.ABOUT' + path = "test_model/this.ABOUT" test_file = get_test_loc(path) abouts = model.About(location=test_file, about_file_path=path) result = get_temp_file() - model.write_output([abouts], result, format='json') + model.write_output([abouts], result, format="json") - expected = get_test_loc('test_model/expected.json') + expected = get_test_loc("test_model/expected.json") check_json(expected, result) + def test_android_module_license(self): + path = "test_model/android/single_license.c.ABOUT" + test_file = get_test_loc(path) + abouts = model.About(location=test_file, about_file_path=path) -class CollectorTest(unittest.TestCase): + parent_dir = get_temp_dir() + abouts.android_module_license(parent_dir) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_PUBLIC_DOMAIN")) + + def test_android_module_multi_licenses(self): + path = "test_model/android/multi_license.c.ABOUT" + test_file = get_test_loc(path) + abouts = model.About(location=test_file, about_file_path=path) + + parent_dir = get_temp_dir() + abouts.android_module_license(parent_dir) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_BSD_NEW")) + assert os.path.exists(os.path.join(parent_dir, "MODULE_LICENSE_BSD_SIMPLIFIED")) + + def test_android_notice(self): + path = "test_model/android/single_license.c.ABOUT" + test_file = get_test_loc(path) + abouts = model.About(location=test_file, about_file_path=path) + + parent_dir = get_temp_dir() + notice_path, notice_context = abouts.android_notice(parent_dir) + expected_path = os.path.join(parent_dir, "NOTICE") + assert os.path.normpath(notice_path) == expected_path + + expected_notice = """Copyright (c) xyz +This component is released to the public domain by the author. + +""" + assert notice_context == expected_notice + + +class CollectorTest(unittest.TestCase): def test_collect_inventory_return_errors(self): - test_loc = get_test_loc('test_model/collect_inventory_errors') + test_loc = get_test_loc("test_model/collect_inventory_errors") errors, _abouts = model.collect_inventory(test_loc) - file_path1 = posixpath.join(test_loc, 'distribute_setup.py') - file_path2 = posixpath.join(test_loc, 'date_test.py') + file_path1 = posixpath.join(test_loc, "distribute_setup.py") + file_path2 = posixpath.join(test_loc, "date_test.py") - err_msg1 = 'non-supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path1 - err_msg2 = 'supported_date_format.ABOUT: Field about_resource: Path %s not found' % file_path2 + err_msg1 = ( + "non-supported_date_format.ABOUT: Field about_resource: Path %s not found" % file_path1 + ) + err_msg2 = ( + "supported_date_format.ABOUT: Field about_resource: Path %s not found" % file_path2 + ) expected_errors = [ - Error(INFO, 'non-supported_date_format.ABOUT: Field date is a custom field.'), - Error(INFO, 'supported_date_format.ABOUT: Field date is a custom field.'), + Error(INFO, "Field ['date'] is a custom field."), Error(INFO, err_msg1), - Error(INFO, err_msg2)] + Error(INFO, err_msg2), + ] assert sorted(expected_errors) == sorted(errors) def test_collect_inventory_with_long_path(self): - test_loc = extract_test_loc('test_model/longpath.zip') + test_loc = extract_test_loc("test_model/longpath.zip") _errors, abouts = model.collect_inventory(test_loc) assert 2 == len(abouts) expected_paths = ( - 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/non-supported_date_format.ABOUT', - 'longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/supported_date_format.ABOUT' + "longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/non-supported_date_format.ABOUT", + "longpath/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/supported_date_format.ABOUT", ) results = [a.about_file_path for a in abouts] assert all(r.endswith(expected_paths) for r in results) - expected_name = ['distribute', 'date_test'] + expected_name = ["distribute", "date_test"] result_name = [a.name.value for a in abouts] assert sorted(expected_name) == sorted(result_name) def test_collect_inventory_can_collect_a_single_file(self): - test_loc = get_test_loc('test_model/single_file/django_snippets_2413.ABOUT') + test_loc = get_test_loc("test_model/single_file/django_snippets_2413.ABOUT") _errors, abouts = model.collect_inventory(test_loc) assert 1 == len(abouts) - expected = ['single_file/django_snippets_2413.ABOUT'] + expected = ["single_file/django_snippets_2413.ABOUT"] result = [a.about_file_path for a in abouts] assert expected == result - def test_collect_inventory_return_no_warnings_and_model_can_uuse_relative_paths(self): - test_loc = get_test_loc('test_model/rel/allAboutInOneDir') + def test_collect_inventory_return_no_warnings_and_model_can_use_relative_paths(self): + test_loc = get_test_loc("test_model/rel/allAboutInOneDir") errors, _abouts = model.collect_inventory(test_loc) expected_errors = [] result = [(level, e) for level, e in errors if level > INFO] assert expected_errors == result def test_collect_inventory_populate_about_file_path(self): - test_loc = get_test_loc('test_model/inventory/complete') + test_loc = get_test_loc("test_model/inventory/complete") errors, abouts = model.collect_inventory(test_loc) assert [] == errors - expected = 'about.ABOUT' + expected = "about.ABOUT" result = abouts[0].about_file_path assert expected == result def test_collect_inventory_with_multi_line(self): - test_loc = get_test_loc('test_model/parse/multi_line_license_expresion.ABOUT') + test_loc = get_test_loc("test_model/parse/multi_line_license_expresion.ABOUT") errors, abouts = model.collect_inventory(test_loc) assert [] == errors expected_lic_url = [ - 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', - 'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0'] + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0", + ] returned_lic_url = abouts[0].license_url.value assert expected_lic_url == returned_lic_url def test_collect_inventory_with_license_expression(self): - test_loc = get_test_loc('test_model/parse/multi_line_license_expresion.ABOUT') + test_loc = get_test_loc("test_model/parse/multi_line_license_expresion.ABOUT") errors, abouts = model.collect_inventory(test_loc) assert [] == errors - expected_lic = 'mit or apache-2.0' + expected_lic = "mit or apache-2.0" returned_lic = abouts[0].license_expression.value assert expected_lic == returned_lic def test_collect_inventory_always_collects_custom_fieldsg(self): - test_loc = get_test_loc('test_model/inventory/custom_fields.ABOUT') + test_loc = get_test_loc("test_model/inventory/custom_fields.ABOUT") errors, abouts = model.collect_inventory(test_loc) - expected_msg1 = 'Field resource is a custom field' - assert len(errors) == 2 - assert expected_msg1 in errors[0].message - # The not supported 'resource' value is collected + expected_msg = "Field ['resource', 'custom_mapping'] is a custom field." + assert len(errors) == 1 + assert expected_msg in errors[0].message + # The value of the custom field: 'resource' is collected assert abouts[0].resource.value def test_collect_inventory_does_not_raise_error_and_maintains_order_on_custom_fields(self): - test_loc = get_test_loc('test_model/inventory/custom_fields2.ABOUT') + test_loc = get_test_loc("test_model/inventory/custom_fields2.ABOUT") errors, abouts = model.collect_inventory(test_loc) - expected_errors = [ - Error(INFO, 'inventory/custom_fields2.ABOUT: Field resource is a custom field.'), - Error(INFO, 'inventory/custom_fields2.ABOUT: Field custom_mapping is a custom field.') - ] + expected_errors = [Error(INFO, "Field ['resource', 'custom_mapping'] is a custom field.")] assert expected_errors == errors - expected = [u'about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n'] + expected = ["about_resource: .\nname: test\nresource: .\ncustom_mapping: test\n"] assert expected == [a.dumps() for a in abouts] def test_parse_license_expression(self): - spec_char, returned_lic = model.parse_license_expression('mit or apache-2.0') - expected_lic = ['mit', 'apache-2.0'] + spec_char, returned_lic, _invalid_lic_exp = model.parse_license_expression( + "mit or apache-2.0" + ) + expected_lic = ["mit", "apache-2.0"] expected_spec_char = [] assert expected_lic == returned_lic assert expected_spec_char == spec_char def test_parse_license_expression_with_special_chara(self): - spec_char, returned_lic = model.parse_license_expression('mit, apache-2.0') + spec_char, returned_lic, _invalid_lic_exp = model.parse_license_expression( + "mit, apache-2.0" + ) expected_lic = [] - expected_spec_char = [','] + expected_spec_char = [","] assert expected_lic == returned_lic assert expected_spec_char == spec_char @@ -1061,100 +1219,172 @@ def test_collect_inventory_works_with_relative_paths(self): # FIXME: This test need to be run under src/attributecode/ # or otherwise it will fail as the test depends on the launching # location - test_loc = get_test_loc('test_model/inventory/relative') + test_loc = get_test_loc("test_model/inventory/relative") # Use '.' as the indication of the current directory - test_loc1 = test_loc + '/./' + test_loc1 = test_loc + "/./" # Use '..' to go back to the parent directory - test_loc2 = test_loc + '/../relative' + test_loc2 = test_loc + "/../relative" errors1, abouts1 = model.collect_inventory(test_loc1) errors2, abouts2 = model.collect_inventory(test_loc2) assert [] == errors1 assert [] == errors2 - expected = 'about.ABOUT' + expected = "about.ABOUT" result1 = abouts1[0].about_file_path result2 = abouts2[0].about_file_path assert expected == result1 assert expected == result2 def test_collect_inventory_basic_from_directory(self): - location = get_test_loc('test_model/inventory/basic') + location = get_test_loc("test_model/inventory/basic") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") expected_errors = [] assert expected_errors == errors - expected = get_test_loc('test_model/inventory/basic/expected.csv') + expected = get_test_loc("test_model/inventory/basic/expected.csv") check_csv(expected, result) def test_collect_inventory_with_about_resource_path_from_directory(self): - location = get_test_loc('test_model/inventory/basic_with_about_resource_path') + location = get_test_loc("test_model/inventory/basic_with_about_resource_path") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") expected_errors = [] assert expected_errors == errors - expected = get_test_loc('test_model/inventory/basic_with_about_resource_path/expected.csv') + expected = get_test_loc("test_model/inventory/basic_with_about_resource_path/expected.csv") check_csv(expected, result) def test_collect_inventory_with_no_about_resource_from_directory(self): - location = get_test_loc('test_model/inventory/no_about_resource_key') - result = get_temp_file() + location = get_test_loc("test_model/inventory/no_about_resource_key") errors, abouts = model.collect_inventory(location) - - model.write_output(abouts, result, format='csv') - - expected_errors = [Error(CRITICAL, 'about/about.ABOUT: Field about_resource is required')] - assert expected_errors == errors + assert errors == [] def test_collect_inventory_complex_from_directory(self): - location = get_test_loc('test_model/inventory/complex') + location = get_test_loc("test_model/inventory/complex") result = get_temp_file() errors, abouts = model.collect_inventory(location) - model.write_output(abouts, result, format='csv') + model.write_output(abouts, result, format="csv") assert all(e.severity == INFO for e in errors) - expected = get_test_loc('test_model/inventory/complex/expected.csv') + expected = get_test_loc("test_model/inventory/complex/expected.csv") check_csv(expected, result, fix_cell_linesep=True, regen=False) def test_collect_inventory_does_not_convert_lf_to_crlf_from_directory(self): - location = get_test_loc('test_model/crlf/about.ABOUT') + location = get_test_loc("test_model/crlf/about.ABOUT") result = get_temp_file() errors, abouts = model.collect_inventory(location) - errors2 = model.write_output(abouts, result, format='csv') - errors.extend(errors2) + model.write_output(abouts, result, format="csv") assert all(e.severity == INFO for e in errors) - expected = get_test_loc('test_model/crlf/expected.csv') + expected = get_test_loc("test_model/crlf/expected.csv") check_csv(expected, result, fix_cell_linesep=True, regen=False) + def test_copy_redist_src_no_structure(self): + test_loc = get_test_loc("test_model/redistribution/") + copy_list = [ + get_test_loc("test_model/redistribution/this.c"), + get_test_loc("test_model/redistribution/test/subdir"), + ] + output = get_temp_dir() -class FetchLicenseTest(unittest.TestCase): + expected_file = ["this.c", "subdir"] + + with_structure = False + err = model.copy_redist_src(copy_list, test_loc, output, with_structure) + + assert err == [] + + from os import listdir + + copied_files = listdir(output) + assert len(expected_file) == len(copied_files) + assert err == [] + for file in expected_file: + assert file in copied_files + + def test_copy_redist_src_with_structure(self): + test_loc = get_test_loc("test_model/redistribution/") + copy_list = [ + get_test_loc("test_model/redistribution/this.c"), + get_test_loc("test_model/redistribution/test/subdir"), + ] + output = get_temp_dir() + + expected_file = ["this.c", "test"] + + with_structure = True + err = model.copy_redist_src(copy_list, test_loc, output, with_structure) + + assert err == [] + + from os import listdir - @mock.patch.object(model, 'urlopen') + copied_files = listdir(output) + assert len(expected_file) == len(copied_files) + assert err == [] + for file in expected_file: + assert file in copied_files + + def test_get_copy_list(self): + location = get_test_loc("test_model/redistribution/") + result = get_temp_file() + errors, abouts = model.collect_inventory(location) + copy_list, err = model.get_copy_list(abouts, location) + assert err == [] + expected = [os.path.join(location, "this.c"), os.path.join(location, "test/subdir")] + if on_windows: + norm_list = [] + for c in copy_list: + norm_list.append(norm(c)) + assert norm_list == expected + else: + assert copy_list == expected + + +class FetchLicenseTest(unittest.TestCase): + @mock.patch.object(model, "get") def test_valid_api_url(self, mock_data): - mock_data.return_value = '' - assert model.valid_api_url('non_valid_url') is False + mock_data.return_value = "" + assert model.valid_api_url("non_valid_url") is False - @mock.patch('attributecode.util.have_network_connection') - @mock.patch('attributecode.model.valid_api_url') - def test_pre_process_and_fetch_license_dict(self, have_network_connection, valid_api_url): + @mock.patch("attributecode.util.have_network_connection") + @mock.patch("attributecode.model.valid_api_url") + def test_pre_process_and_fetch_license_dict_dje(self, have_network_connection, valid_api_url): have_network_connection.return_value = True + valid_api_url.return_value = False + error_msg = ( + "Network problem. Please check your Internet connection. License generation is skipped." + ) + expected = ({}, [Error(ERROR, error_msg)]) + assert model.pre_process_and_fetch_license_dict([]) == expected + valid_api_url.return_value = True + expected = ({}, []) + assert model.pre_process_and_fetch_license_dict([]) == expected + + @mock.patch("attributecode.util.have_network_connection") + @mock.patch("attributecode.model.valid_api_url") + def test_pre_process_and_fetch_license_dict_licensedb( + self, have_network_connection, valid_api_url + ): + have_network_connection.return_value = False valid_api_url.return_value = False error_msg = ( - 'Network problem. Please check your Internet connection. ' - 'License generation is skipped.') + "Network problem. Please check your Internet connection. License generation is skipped." + ) expected = ({}, [Error(ERROR, error_msg)]) - assert model.pre_process_and_fetch_license_dict([], '', '') == expected + assert model.pre_process_and_fetch_license_dict([]) == expected + have_network_connection.return_value = True valid_api_url.return_value = True expected = ({}, []) - assert model.pre_process_and_fetch_license_dict([], '', '') == expected + + assert model.pre_process_and_fetch_license_dict([]) == expected diff --git a/tests/test_transform.py b/tests/test_transform.py index d382bfa8..ab3c3a65 100644 --- a/tests/test_transform.py +++ b/tests/test_transform.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,32 +14,518 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - from collections import OrderedDict import unittest -from testing_utils import get_temp_dir from testing_utils import get_test_loc -from attributecode import ERROR -from attributecode import INFO -from attributecode import CRITICAL -from attributecode import Error -from attributecode import gen -from attributecode.transform import read_csv_rows +from attributecode.transform import check_duplicate_fields from attributecode.transform import transform_data +from attributecode.transform import normalize_dict_data +from attributecode.transform import strip_trailing_fields_csv +from attributecode.transform import strip_trailing_fields_json from attributecode.transform import Transformer +from attributecode.transform import read_csv_rows, read_excel, read_json +from attributecode.transform import transform_csv, transform_excel, transform_json class TransformTest(unittest.TestCase): + def test_transform_data_new_col(self): + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration_new_cols") + transformer = Transformer.from_file(configuration) + + data, err = transform_data(data, transformer) + + expected_data = [ + dict( + OrderedDict( + [ + ("path", "/tmp/test.c"), + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ) + ] + assert len(data) == len(expected_data) + for d in data: + assert dict(d) in expected_data + def test_transform_data(self): - test_file = get_test_loc('test_transform/input.csv') - configuration = get_test_loc('test_transform/configuration') - rows = read_csv_rows(test_file) + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") transformer = Transformer.from_file(configuration) - col_name, data, err = transform_data(rows, transformer) - expect = [u'about_resource', u'name'] - assert col_name == expect + + data, err = transform_data(data, transformer) + + expect_name = ["about_resource", "name", "version"] + expected_data = [ + dict( + OrderedDict( + [("about_resource", "/tmp/test.c"), ("name", "test.c"), ("version", "1")] + ) + ) + ] + + assert len(data) == len(expected_data) + for d in data: + assert dict(d) in expected_data + + def test_transform_data_mutli_rows(self): + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("Confirmed Version", "v0.01"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/tmp/tmp.h"), + ("Component", "tmp.h"), + ("Confirmed Version", None), + ] + ), + ] + configuration = get_test_loc("test_transform/configuration2") + transformer = Transformer.from_file(configuration) + + data, err = transform_data(data, transformer) + + expect_name = ["about_resource", "name", "version"] + expected_data = [ + dict( + OrderedDict( + [("about_resource", "/tmp/test.c"), ("name", "test.c"), ("version", "v0.01")] + ) + ), + dict( + OrderedDict( + [("about_resource", "/tmp/tmp.h"), ("name", "tmp.h"), ("version", None)] + ) + ), + ] + + assert len(data) == len(expected_data) + for d in data: + assert dict(d) in expected_data + + def test_normalize_dict_data_scancode(self): + test_file = get_test_loc("test_transform/input_scancode.json") + json_data = read_json(test_file) + data = normalize_dict_data(json_data) + expected_data = [ + OrderedDict( + [ + ("path", "samples"), + ("type", "directory"), + ("name", "samples"), + ("base_name", "samples"), + ("extension", ""), + ("size", 0), + ("date", None), + ("sha1", None), + ("md5", None), + ("mime_type", None), + ("file_type", None), + ("programming_language", None), + ("is_binary", False), + ("is_text", False), + ("is_archive", False), + ("is_media", False), + ("is_source", False), + ("is_script", False), + ("licenses", []), + ("license_expressions", []), + ("copyrights", []), + ("holders", []), + ("authors", []), + ("packages", []), + ("emails", []), + ("urls", []), + ("files_count", 33), + ("dirs_count", 10), + ("size_count", 1161083), + ("scan_errors", []), + ] + ) + ] + assert data == expected_data + + def test_normalize_dict_data_json(self): + json_data = OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.2.3"), + ("note", "test"), + ("temp", "foo"), + ] + ) + data = normalize_dict_data(json_data) + expected_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.2.3"), + ("note", "test"), + ("temp", "foo"), + ] + ) + ] + assert data == expected_data + + def test_normalize_dict_data_json_array(self): + json_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.0"), + ("temp", "fpp"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit1/"), + ("Component", "AboutCode-toolkit1"), + ("version", "1.1"), + ("temp", "foo"), + ] + ), + ] + data = normalize_dict_data(json_data) + expected_data = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("version", "1.0"), + ("temp", "fpp"), + ] + ), + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit1/"), + ("Component", "AboutCode-toolkit1"), + ("version", "1.1"), + ("temp", "foo"), + ] + ), + ] + assert data == expected_data + + def test_check_duplicate_fields(self): + field_name = ["path", "name", "path", "version"] + expected = ["path"] + dups = check_duplicate_fields(field_name) + assert dups == expected + + def test_strip_trailing_fields_csv(self): + test = ["about_resource", "name ", " version "] + expected = ["about_resource", "name", "version"] + result = strip_trailing_fields_csv(test) + assert result == expected + + def test_strip_trailing_fields_json(self): + test = [ + OrderedDict( + [("about_resource", "/this.c"), ("name ", "this.c"), (" version ", "0.11.0")] + ) + ] + expected = [ + OrderedDict([("about_resource", "/this.c"), ("name", "this.c"), ("version", "0.11.0")]) + ] + result = strip_trailing_fields_json(test) + assert result == expected + + def test_read_excel(self): + test_file = get_test_loc("test_transform/simple.xlsx") + error, data = read_excel(test_file) + assert not error + expected = [ + OrderedDict( + [("about_resource", "/test.c"), ("name", "test.c"), ("license_expression", "mit")] + ), + OrderedDict( + [ + ("about_resource", "/test2.c"), + ("name", "test2.c"), + ("license_expression", "mit and apache-2.0"), + ] + ), + ] + assert data == expected + + def test_read_csv_rows(self): + test_file = get_test_loc("test_transform/simple.csv") + data = read_csv_rows(test_file) + expected = [ + ["about_resource", "name", "license_expression"], + ["/test.c", "test.c", "mit"], + ["/test2.c", "test2.c", "mit and apache-2.0"], + ] + assert list(data) == expected + + def test_transform_csv(self): + test_file = get_test_loc("test_transform/input.csv") + data, err = transform_csv(test_file) + expected = [ + { + "Directory/Filename": "/aboutcode-toolkit/", + "Component": "AboutCode-toolkit", + "Confirmed Version": "123", + "notes": "", + } + ] + assert len(err) == 0 + assert data == expected + + def test_transform_excel(self): + test_file = get_test_loc("test_transform/input.xlsx") + data, err = transform_excel(test_file) + expected = [ + OrderedDict( + [ + ("Directory/Filename", "/aboutcode-toolkit/"), + ("Component", "AboutCode-toolkit"), + ("Confirmed Version", 123), + ("notes", ""), + ] + ) + ] + assert len(err) == 0 + assert data == expected + + def test_transform_json(self): + test_file = get_test_loc("test_transform/input.json") + data, err = transform_json(test_file) + expected = [ + { + "Directory/Filename": "/aboutcode-toolkit/", + "Component": "AboutCode-toolkit", + "Confirmed Version": "123", + "notes": "", + } + ] + assert len(err) == 0 + assert data == expected + + def test_apply_renamings(self): + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") + transformer = Transformer.from_file(configuration) + + expected = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + renamed_field_data = transformer.apply_renamings(data) + assert renamed_field_data == expected + + def test_apply_renamings_nested_list(self): + data = [ + { + "path": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] + configuration = get_test_loc("test_transform/configuration3") + transformer = Transformer.from_file(configuration) + + expected = [ + { + "about_resource": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score_renamed": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score_renamed": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] + updated_data = transformer.apply_renamings(data) + assert updated_data == expected + + def test_filter_excluded(self): + data = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") + transformer = Transformer.from_file(configuration) + + expected = [ + OrderedDict( + [ + ("Directory/Filename", "/tmp/test.c"), + ("Component", "test.c"), + ("version", "1"), + ("notes", "test"), + ] + ) + ] + updated_data = transformer.filter_excluded(data) + assert updated_data == expected + + def test_filter_excluded_nested_list(self): + data = [ + { + "path": "samples/JGroups-error.log", + "type": "file", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + { + "score": 90.0, + "start_line": 4, + "end_line": 4, + "license_expression": "apache-1.1", + }, + { + "score": 100.0, + "start_line": 5, + "end_line": 5, + "license_expression": "apache-2.0", + }, + ], + } + ], + } + ] + configuration = get_test_loc("test_transform/configuration3") + transformer = Transformer.from_file(configuration) + + expected = [ + { + "path": "samples/JGroups-error.log", + "name": "JGroups-error.log", + "license_detections": [ + { + "license_expression": "apache-1.1 AND apache-2.0", + "matches": [ + {"score": 90.0, "end_line": 4, "license_expression": "apache-1.1"}, + {"score": 100.0, "end_line": 5, "license_expression": "apache-2.0"}, + ], + } + ], + } + ] + updated_data = transformer.filter_excluded(data) + assert updated_data == expected + + def test_filter_fields(self): + data = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("notes", "test"), + ("temp", "foo"), + ] + ) + ] + configuration = get_test_loc("test_transform/configuration") + transformer = Transformer.from_file(configuration) + + updated_data = transformer.filter_fields(data) + + expected = [ + OrderedDict( + [ + ("about_resource", "/tmp/test.c"), + ("name", "test.c"), + ("version", "1"), + ("temp", "foo"), + ] + ) + ] + + for d in updated_data: + assert dict(d) in expected diff --git a/tests/test_util.py b/tests/test_util.py index c7456f63..3bf14045 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2014-2019 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,11 +14,6 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - -from collections import OrderedDict import string import unittest @@ -26,6 +21,7 @@ from testing_utils import extract_test_loc from testing_utils import get_test_loc +from testing_utils import get_temp_dir from testing_utils import on_posix from testing_utils import on_windows @@ -36,154 +32,154 @@ class TestResourcePaths(unittest.TestCase): - def test_resource_name(self): - expected = 'first' - result = util.resource_name('some/things/first') + expected = "first" + result = util.resource_name("some/things/first") assert expected == result def test_resource_name_with_extension(self): - expected = 'first.ABOUT' - result = util.resource_name('/some/things/first.ABOUT') + expected = "first.ABOUT" + result = util.resource_name("/some/things/first.ABOUT") assert expected == result def test_resource_name_for_dir(self): - expected = 'first' - result = util.resource_name('some/things/first/') + expected = "first" + result = util.resource_name("some/things/first/") assert expected == result def test_resource_name_windows(self): - expected = r'first.' - result = util.resource_name(r'c:\some\things\first.') + expected = r"first." + result = util.resource_name(r"c:\some\things\first.") assert expected == result def test_resource_name_mixed_windows_posix(self): - expected = r'first' - result = util.resource_name(r'c:\some/things\first') + expected = r"first" + result = util.resource_name(r"c:\some/things\first") assert expected == result def test_resource_name_double_slash(self): - expected = 'first' - result = util.resource_name(r'some\thi ngs//first') + expected = "first" + result = util.resource_name(r"some\thi ngs//first") assert expected == result def test_resource_name_punctuation(self): - expected = '_$asafg:' - result = util.resource_name('%6571351()2/75612$/_$asafg:') + expected = "_$asafg:" + result = util.resource_name("%6571351()2/75612$/_$asafg:") assert expected == result def test_resource_name_simple_slash(self): - expected = '' - result = util.resource_name('/') + expected = "" + result = util.resource_name("/") assert expected == result def test_resource_name_spaces(self): - expected = '' - result = util.resource_name('/ / ') + expected = "" + result = util.resource_name("/ / ") assert expected == result def test_resource_name_does_not_recurse_infinitely(self): - expected = '' - result = util.resource_name(' / ') + expected = "" + result = util.resource_name(" / ") assert expected == result def test_to_posix_from_win(self): - test = r'c:\this\that' - expected = 'c:/this/that' + test = r"c:\this\that" + expected = "c:/this/that" result = util.to_posix(test) assert expected == result def test_to_posix_from_posix(self): - test = r'/this/that' - expected = '/this/that' + test = r"/this/that" + expected = "/this/that" result = util.to_posix(test) assert expected == result def test_to_posix_from_mixed(self): - test = r'/this/that\this' - expected = '/this/that/this' + test = r"/this/that\this" + expected = "/this/that/this" result = util.to_posix(test) assert expected == result def test_to_native_from_win(self): - test = r'c:\this\that' + test = r"c:\this\that" if on_posix: - expected = 'c:/this/that' + expected = "c:/this/that" else: expected = test result = util.to_native(test) assert expected == result def test_to_native_from_posix(self): - test = r'/this/that' + test = r"/this/that" if on_windows: - expected = r'\this\that' + expected = r"\this\that" else: expected = test result = util.to_native(test) assert expected == result def test_to_native_from_mixed(self): - test = r'/this/that\this' + test = r"/this/that\this" if on_windows: - expected = r'\this\that\this' + expected = r"\this\that\this" else: - expected = r'/this/that/this' + expected = r"/this/that/this" result = util.to_native(test) assert expected == result def test_invalid_chars_with_valid_chars(self): - name = string.digits + string.ascii_letters + '_-.+' + name = string.digits + string.ascii_letters + "_-.+()~[]{}@%!$," result = util.invalid_chars(name) expected = [] assert expected == result def test_space_is_valid_chars(self): - result = util.invalid_chars(' ') + result = util.invalid_chars(" ") expected = [] assert expected == result def test_invalid_chars_with_invalid_in_name_and_dir(self): - result = util.invalid_chars('_$as/afg:') - expected = [':'] + result = util.invalid_chars("_$as/afg:") + expected = [":"] assert expected == result def test_invalid_chars_in_file_name(self): - name = '%657!1351()275612$_$asafg:~|[]{}+-.' + name = "%657!1351()275612$_$asafg:~|[]{}+-." result = util.invalid_chars(name) - expected = ['%', '!', '$', '$', ':'] + expected = [":", "|"] assert expected == result def test_invalid_chars_with_space_is_valid(self): - result = util.invalid_chars('_ Hello') + result = util.invalid_chars("_ Hello") expected = [] assert expected == result def test_check_file_names_with_dupes_return_errors(self): - paths = ['some/path', 'some/PAth'] + paths = ["some/path", "some/PAth"] result = util.check_file_names(paths) expected = [ Error( CRITICAL, - "Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name") - ] + "Duplicate files: 'some/PAth' and 'some/path' have the same case-insensitive file name", + ) + ] assert expected == result def test_check_file_names_without_dupes_return_no_error(self): - paths = ['some/path', - 'some/otherpath'] + paths = ["some/path", "some/otherpath"] result = util.check_file_names(paths) expected = [] assert expected == result def test_check_file_names_with_no_invalid_char_return_no_error(self): paths = [ - 'locations/file', - 'locations/file1', - 'locations/file2', - 'locations/dir1/file2', - 'locations/dir1/dir2/file1', - 'locations/dir2/file1'] + "locations/file", + "locations/file1", + "locations/file2", + "locations/dir1/file2", + "locations/dir1/dir2/file1", + "locations/dir2/file1", + ] expected = [] result = util.check_file_names(paths) @@ -191,307 +187,409 @@ def test_check_file_names_with_no_invalid_char_return_no_error(self): def test_check_file_names_with_invalid_chars_return_errors(self): paths = [ - 'locations/file', - 'locations/file with space', - 'locations/dir1/dir2/file1', - 'locations/dir2/file1', - 'Accessibilité/ périmètre' + "locations/file", + "locations/file with space", + "locations/dir1/dir2/file1", + "locations/dir2/file1", + "Accessibilité/ périmètre", + "locations/in:valid", ] import sys + if sys.version_info[0] < 3: # python2 - expected = [Error(CRITICAL, b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'")] + expected = [ + Error( + CRITICAL, + b"Invalid characters '\xe9\xe8' in file name at: 'Accessibilit\xe9/ p\xe9rim\xe8tre'", + ) + ] else: - expected = [Error(CRITICAL, "Invalid characters 'éè' in file name at: 'Accessibilité/ périmètre'")] + expected = [ + Error(CRITICAL, "Invalid characters ':' in file name at: 'locations/in:valid'") + ] result = util.check_file_names(paths) assert expected[0].message == result[0].message assert expected == result def test_is_about_file(self): - assert util.is_about_file('test.About') - assert util.is_about_file('test2.aboUT') - assert not util.is_about_file('no_about_ext.something') - assert not util.is_about_file('about') - assert not util.is_about_file('about.txt') + assert util.is_about_file("test.About") + assert util.is_about_file("test2.aboUT") + assert not util.is_about_file("no_about_ext.something") + assert not util.is_about_file("about") + assert not util.is_about_file("about.txt") def test_is_about_file_is_false_if_only_bare_extension(self): - assert not util.is_about_file('.ABOUT') + assert not util.is_about_file(".ABOUT") def test_get_relative_path(self): - test = [('/some/path', '/some/path/file', 'file'), - ('path', '/path/file', 'file'), - ('/path', '/path/file', 'file'), - ('/path/', '/path/file/', 'file'), - ('/path/', 'path/', 'path'), - ('/p1/p2/p3', '/p1/p2//p3/file', 'file'), - (r'c:\some/path', 'c:/some/path/file', 'file'), - (r'c:\\some\\path\\', 'c:/some/path/file', 'file'), - ] + test = [ + ("/some/path", "/some/path/file", "file"), + ("path", "/path/file", "file"), + ("/path", "/path/file", "file"), + ("/path/", "/path/file/", "file"), + ("/path/", "path/", "path"), + ("/p1/p2/p3", "/p1/p2//p3/file", "file"), + (r"c:\some/path", "c:/some/path/file", "file"), + (r"c:\\some\\path\\", "c:/some/path/file", "file"), + ] for base_loc, full_loc, expected in test: result = util.get_relative_path(base_loc, full_loc) assert expected == result def test_get_relative_path_with_same_path_twice(self): - test = [('/some/path/file', 'path/file'), - ('/path/file', 'path/file'), - ('/path/file/', 'path/file'), - ('path/', 'path'), - ('/p1/p2//p3/file', 'p3/file'), - ('c:/some/path/file', 'path/file'), - (r'c:\\some\\path\\file', 'path/file'), - ] + test = [ + ("/some/path/file", "path/file"), + ("/path/file", "path/file"), + ("/path/file/", "path/file"), + ("path/", "path"), + ("/p1/p2//p3/file", "p3/file"), + ("c:/some/path/file", "path/file"), + (r"c:\\some\\path\\file", "path/file"), + ] for loc, expected in test: result = util.get_relative_path(loc, loc) assert expected == result class TestGetLocations(unittest.TestCase): - def test_get_locations(self): - test_dir = get_test_loc('test_util/about_locations') - expected = sorted([ - 'file with_spaces.ABOUT', - 'file1', - 'file2', - 'dir1/file2', - 'dir1/file2.aBout', - 'dir1/dir2/file1.about', - 'dir2/file1']) + test_dir = get_test_loc("test_util/about_locations") + expected = sorted( + [ + "file with_spaces.ABOUT", + "file1", + "file2", + "dir1/file2", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + "dir2/file1", + ] + ) result = sorted(util.get_locations(test_dir)) - result = [l.partition('/about_locations/')[-1] for l in result] + result = [l.partition("/about_locations/")[-1] for l in result] assert expected == result def test_get_about_locations(self): - test_dir = get_test_loc('test_util/about_locations') - expected = sorted([ - 'file with_spaces.ABOUT', - 'dir1/file2.aBout', - 'dir1/dir2/file1.about', - ]) + test_dir = get_test_loc("test_util/about_locations") + expected = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + ] + ) result = sorted(util.get_about_locations(test_dir)) - result = [l.partition('/about_locations/')[-1] for l in result] - assert expected == result + result = [l.partition("/about_locations/")[-1] for l in result] + assert expected == result + + def test_get_about_locations_with_exclude(self): + test_dir = get_test_loc("test_util/about_locations") + exclude1 = ("dir*",) + exclude2 = ("*dir2*",) + exclude3 = ("*test*",) + exclude4 = ("dir1/",) + expected1 = sorted( + [ + "file with_spaces.ABOUT", + ] + ) + expected2 = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + ] + ) + expected3 = sorted( + [ + "file with_spaces.ABOUT", + "dir1/file2.aBout", + "dir1/dir2/file1.about", + ] + ) + expected4 = sorted( + [ + "file with_spaces.ABOUT", + ] + ) + + result1 = sorted(util.get_about_locations(test_dir, exclude1)) + result2 = sorted(util.get_about_locations(test_dir, exclude2)) + result3 = sorted(util.get_about_locations(test_dir, exclude3)) + result4 = sorted(util.get_about_locations(test_dir, exclude4)) + + result1 = [l.partition("/about_locations/")[-1] for l in result1] + result2 = [l.partition("/about_locations/")[-1] for l in result2] + result3 = [l.partition("/about_locations/")[-1] for l in result3] + result4 = [l.partition("/about_locations/")[-1] for l in result4] + + assert expected1 == result1 + assert expected2 == result2 + assert expected3 == result3 + assert expected4 == result4 def test_get_locations_can_yield_a_single_file(self): - test_file = get_test_loc('test_util/about_locations/file with_spaces.ABOUT') + test_file = get_test_loc("test_util/about_locations/file with_spaces.ABOUT") result = list(util.get_locations(test_file)) assert 1 == len(result) def test_get_about_locations_for_about(self): - location = get_test_loc('test_util/get_about_locations') + location = get_test_loc("test_util/get_about_locations") result = list(util.get_about_locations(location)) - expected = 'get_about_locations/about.ABOUT' + expected = "get_about_locations/about.ABOUT" assert result[0].endswith(expected) # FIXME: these are not very long/deep paths def test_get_locations_with_very_long_path(self): longpath = ( - 'longpath' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' - '/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1' + "longpath" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" + "/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1/longpath1" ) - test_loc = extract_test_loc('test_util/longpath.zip') + test_loc = extract_test_loc("test_util/longpath.zip") result = list(util.get_locations(test_loc)) assert any(longpath in r for r in result) class TestCsv(unittest.TestCase): - def test_load_csv_without_mapping(self): - test_file = get_test_loc('test_util/csv/about.csv') - expected = [OrderedDict([ - ('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) + test_file = get_test_loc("test_util/csv/about.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) ] result = util.load_csv(test_file) assert expected == result def test_load_csv_load_rows(self): - test_file = get_test_loc('test_util/csv/about.csv') - expected = [OrderedDict([ - ('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) + test_file = get_test_loc("test_util/csv/about.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) ] result = util.load_csv(test_file) assert expected == result def test_load_csv_does_convert_column_names_to_lowercase(self): - test_file = get_test_loc('test_util/csv/about_key_with_upper_case.csv') - expected = [OrderedDict( - [('about_file', 'about.ABOUT'), - ('about_resource', '.'), - ('name', 'ABOUT tool'), - ('version', '0.8.1')]) - ] + test_file = get_test_loc("test_util/csv/about_key_with_upper_case.csv") + expected = [ + dict( + [ + ("about_file", "about.ABOUT"), + ("about_resource", "."), + ("name", "ABOUT tool"), + ("version", "0.8.1"), + ] + ) + ] result = util.load_csv(test_file) assert expected == result - def test_format_about_dict_for_csv_output(self): - about = [OrderedDict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', [u'test.c']), - (u'name', u'AboutCode-toolkit'), - (u'license_expression', u'mit AND bsd-new'), - (u'license_key', [u'mit', u'bsd-new'])])] + def test_format_about_dict_output(self): + about = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", ["test.c"]), + ("name", "AboutCode-toolkit"), + ("license_expression", "mit AND bsd-new"), + ("license_key", ["mit", "bsd-new"]), + ] + ) + ] - expected = [OrderedDict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', u'test.c'), - (u'name', u'AboutCode-toolkit'), - (u'license_expression', u'mit AND bsd-new'), - (u'license_key', u'mit\nbsd-new')])] + expected = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", "test.c"), + ("name", "AboutCode-toolkit"), + ("license_expression", "mit AND bsd-new"), + ("license_key", "mit\nbsd-new"), + ] + ) + ] - output = util.format_about_dict_for_csv_output(about) + output = util.format_about_dict_output(about) assert output == expected + def test_load_csv_microsoft_utf_8(self): + test_file = get_test_loc("test_util/csv/test_ms_utf8.csv") + expected = [dict([("about_resource", "/myFile"), ("name", "myName")])] + result = util.load_csv(test_file) + assert expected == result + + def test_load_csv_utf_8(self): + test_file = get_test_loc("test_util/csv/test_utf8.csv") + expected = [dict([("about_resource", "/myFile"), ("name", "\u540d")])] + result = util.load_csv(test_file) + assert expected == result -class TestJson(unittest.TestCase): +class TestJson(unittest.TestCase): def test_load_json(self): - test_file = get_test_loc('test_util/json/expected.json') - expected = [OrderedDict([ - ('about_file_path', '/load/this.ABOUT'), - ('about_resource', '.'), - ('name', 'AboutCode'), - ('version', '0.11.0')]) + test_file = get_test_loc("test_util/json/expected.json") + expected = [ + dict( + [ + ("about_file_path", "/load/this.ABOUT"), + ("about_resource", "."), + ("name", "AboutCode"), + ("version", "0.11.0"), + ] + ) ] result = util.load_json(test_file) assert expected == result - def test_load_json2(self): - test_file = get_test_loc('test_util/json/expected_need_mapping.json') - expected = [dict(OrderedDict([ - ('about_file', '/load/this.ABOUT'), - ('about_resource', '.'), - ('version', '0.11.0'), - ('name', 'AboutCode'), - ]) - )] + def test_load_json_multi_entries(self): + test_file = get_test_loc("test_util/json/multi_entries.json") + expected = [ + dict( + [ + ("about_file_path", "/load/this.ABOUT"), + ("about_resource", "."), + ("name", "AboutCode"), + ("version", "0.11.0"), + ] + ), + dict( + [("about_file_path", "/load/that.ABOUT"), ("about_resource", "."), ("name", "that")] + ), + ] result = util.load_json(test_file) assert expected == result - def test_load_non_list_json(self): - test_file = get_test_loc('test_util/json/not_a_list_need_mapping.json') - # FIXME: why this dict nesting?? - expected = [dict(OrderedDict([ - ('about_resource', '.'), - ('name', 'AboutCode'), - ('path', '/load/this.ABOUT'), - ('version', '0.11.0'), - ]) - )] + def test_load_json2(self): + test_file = get_test_loc("test_util/json/expected_need_mapping.json") + expected = [ + dict( + dict( + [ + ("about_file", "/load/this.ABOUT"), + ("about_resource", "."), + ("version", "0.11.0"), + ("name", "AboutCode"), + ] + ) + ) + ] result = util.load_json(test_file) assert expected == result - def test_load_non_list_json2(self): - test_file = get_test_loc('test_util/json/not_a_list.json') - expected = [OrderedDict([ - ('about_file_path', '/load/this.ABOUT'), - ('version', '0.11.0'), - ('about_resource', '.'), - ('name', 'AboutCode'), - ]) + def test_load_non_list_json(self): + test_file = get_test_loc("test_util/json/not_a_list_need_mapping.json") + expected = [ + { + "path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0", + } ] result = util.load_json(test_file) assert expected == result - def test_load_json_from_abc_mgr(self): - test_file = get_test_loc('test_util/json/aboutcode_manager_exported.json') - expected = [dict(OrderedDict([ - ('license_expression', 'apache-2.0'), - ('copyright', 'Copyright (c) 2017 nexB Inc.'), - ('licenses', [{'key':'apache-2.0'}]), - ('copyrights', [{'statements':['Copyright (c) 2017 nexB Inc.']}]), - ('path', 'ScanCode'), - ('review_status', 'Analyzed'), - ('name', 'ScanCode'), - ('version', '2.2.1'), - ('owner', 'nexB Inc.'), - ('code_type', 'Source'), - ('is_modified', False), - ('is_deployed', False), - ('feature', ''), - ('purpose', ''), - ('homepage_url', None), - ('download_url', None), - ('license_url', None), - ('notice_url', None), - ('programming_language', 'Python'), - ('notes', ''), - ('fileId', 8458), - ]))] + def test_load_non_list_json2(self): + test_file = get_test_loc("test_util/json/not_a_list.json") + expected = [ + { + "about_file_path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0", + } + ] result = util.load_json(test_file) assert expected == result def test_load_json_from_scancode(self): - test_file = get_test_loc('test_util/json/scancode_info.json') - expected = [dict(OrderedDict([ - ('type', 'file'), - ('name', 'Api.java'), - ('path', 'Api.java'), - ('base_name', 'Api'), - ('extension', '.java'), - ('size', 5074), - ('date', '2017-07-15'), - ('sha1', 'c3a48ec7e684a35417241dd59507ec61702c508c'), - ('md5', '326fb262bbb9c2ce32179f0450e24601'), - ('mime_type', 'text/plain'), - ('file_type', 'ASCII text'), - ('programming_language', 'Java'), - ('is_binary', False), - ('is_text', True), - ('is_archive', False), - ('is_media', False), - ('is_source', True), - ('is_script', False), - ('files_count', 0), - ('dirs_count', 0), - ('size_count', 0), - ('scan_errors', []), - ]))] - result = util.load_json(test_file) + test_file = get_test_loc("test_util/json/scancode_info.json") + expected = [ + { + "about_resource": "lic.txt", + "name": "lic.txt", + "type": "file", + "base_name": "lic", + "extension": ".txt", + "size": 1463, + "date": "2023-07-26", + "sha1": "bb3f381f9ec25416c0c3b4628f7f6b923ced040f", + "md5": "63f9ec8c32874a5d987d78b9a730a6b8", + "sha256": "d71777b3dc333f540a871bf2ef6380e646a10f2ac1f077ce4f34326e16fb6995", + "mime_type": "text/plain", + "file_type": "ASCII text, with very long lines", + "programming_language": None, + "is_binary": False, + "is_text": True, + "is_archive": False, + "is_media": False, + "is_source": False, + "is_script": False, + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [], + } + ] + result = util.load_scancode_json(test_file) assert expected == result def test_format_about_dict_for_json_output(self): - about = [OrderedDict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', OrderedDict([(u'test.c', None)])), - (u'name', u'AboutCode-toolkit'), - (u'license_key', [u'mit', u'bsd-new'])])] - - expected = [OrderedDict([ - (u'about_file_path', u'/input/about1.ABOUT'), - (u'about_resource', u'test.c'), - (u'name', u'AboutCode-toolkit'), - (u'licenses', [ - OrderedDict([(u'key', u'mit')]), - OrderedDict([(u'key', u'bsd-new')])])])] + about = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", dict([("test.c", None)])), + ("name", "AboutCode-toolkit"), + ("license_key", ["mit", "bsd-new"]), + ] + ) + ] + + expected = [ + dict( + [ + ("about_file_path", "/input/about1.ABOUT"), + ("about_resource", "test.c"), + ("name", "AboutCode-toolkit"), + ("licenses", [dict([("key", "mit")]), dict([("key", "bsd-new")])]), + ] + ) + ] output = util.format_about_dict_for_json_output(about) assert output == expected class TestMiscUtils(unittest.TestCase): - def test_load_yaml_about_file_with_no_dupe(self): - test = ''' + test = """ name: test license_expression: mit notes: dup key here - ''' + """ saneyaml.load(test, allow_duplicate_keys=False) def test_load_yaml_about_file_raise_exception_on__duplicate(self): - test = ''' + test = """ name: test notes: some notes notes: dup key here @@ -499,15 +597,15 @@ def test_load_yaml_about_file_raise_exception_on__duplicate(self): notes: dup key here license_expression: mit notes: dup key here - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') - except saneyaml.UnsupportedYamlFeatureError as e : - assert 'Duplicate key in YAML source: notes' == str(e) + self.fail("Exception not raised") + except saneyaml.UnsupportedYamlFeatureError as e: + assert "Duplicate key in YAML source: notes" == str(e) def test_load_yaml_about_file_raise_exception_on_invalid_yaml_ignore_non_key_line(self): - test = ''' + test = """ name: test - notes: some notes - notes: dup key here @@ -516,15 +614,15 @@ def test_load_yaml_about_file_raise_exception_on_invalid_yaml_ignore_non_key_lin notes: dup key here license_expression: mit notes dup key here - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') + self.fail("Exception not raised") except Exception: pass def test_load_yaml_about_file_with_multiline(self): - test = ''' + test = """ name: test owner: test notes: | @@ -534,57 +632,70 @@ def test_load_yaml_about_file_with_multiline(self): notes: continuation line description: sample - ''' + """ try: saneyaml.load(test, allow_duplicate_keys=False) - self.fail('Exception not raised') - except saneyaml.UnsupportedYamlFeatureError as e : + self.fail("Exception not raised") + except saneyaml.UnsupportedYamlFeatureError as e: # notes: exceptio is rasied only for the first dupe - assert 'Duplicate key in YAML source: owner' == str(e) + assert "Duplicate key in YAML source: owner" == str(e) def test_ungroup_licenses(self): about = [ - OrderedDict([ - (u'key', u'mit'), - (u'name', u'MIT License'), - (u'file', u'mit.LICENSE'), - (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit')]), - OrderedDict([ - (u'key', u'bsd-new'), - (u'name', u'BSD-3-Clause'), - (u'file', u'bsd-new.LICENSE'), - (u'url', u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new')]) + dict( + [ + ("key", "mit"), + ("name", "MIT License"), + ("file", "mit.LICENSE"), + ("url", "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit"), + ("spdx_license_key", "MIT"), + ] + ), + dict( + [ + ("key", "bsd-new"), + ("name", "BSD-3-Clause"), + ("file", "bsd-new.LICENSE"), + ("url", "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new"), + ("spdx_license_key", "BSD-3-Clause"), + ] + ), ] - expected_lic_key = [u'mit', u'bsd-new'] - expected_lic_name = [u'MIT License', u'BSD-3-Clause'] - expected_lic_file = [u'mit.LICENSE', u'bsd-new.LICENSE'] + expected_lic_key = ["mit", "bsd-new"] + expected_lic_name = ["MIT License", "BSD-3-Clause"] + expected_lic_file = ["mit.LICENSE", "bsd-new.LICENSE"] expected_lic_url = [ - u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit', - u'https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new'] - lic_key, lic_name, lic_file, lic_url = util.ungroup_licenses(about) + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit", + "https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new", + ] + expected_spdx = ["MIT", "BSD-3-Clause"] + lic_key, lic_name, lic_file, lic_url, spdx_lic_key, lic_score, _matched_text = ( + util.ungroup_licenses(about) + ) assert expected_lic_key == lic_key assert expected_lic_name == lic_name assert expected_lic_file == lic_file assert expected_lic_url == lic_url + assert expected_spdx == spdx_lic_key def test_unique_does_deduplicate_and_keep_ordering(self): - items = ['a', 'b', 'd', 'b', 'c', 'a'] - expected = ['a', 'b', 'd', 'c'] + items = ["a", "b", "d", "b", "c", "a"] + expected = ["a", "b", "d", "c"] results = util.unique(items) assert expected == results def test_unique_can_handle_About_object(self): - base_dir = 'some_dir' + base_dir = "some_dir" test = { - 'about_resource': '.', - 'author': '', - 'copyright': 'Copyright (c) 2013-2014 nexB Inc.', - 'custom1': 'some custom', - 'custom_empty': '', - 'description': 'AboutCode is a tool\nfor files.', - 'license': 'apache-2.0', - 'name': 'AboutCode', - 'owner': 'nexB Inc.' + "about_resource": ".", + "author": "", + "copyright": "Copyright (c) 2013-2014 nexB Inc.", + "custom1": "some custom", + "custom_empty": "", + "description": "AboutCode is a tool\nfor files.", + "license": "apache-2.0", + "name": "AboutCode", + "owner": "nexB Inc.", } a = model.About() @@ -594,9 +705,78 @@ def test_unique_can_handle_About_object(self): c.load_dict(test, base_dir) b = model.About() - test.update(dict(about_resource='asdasdasd')) + test.update(dict(about_resource="asdasdasd")) b.load_dict(test, base_dir) abouts = [a, b] results = util.unique(abouts) assert [a] == results + + def test_copy_license_notice_files(self): + base_dir = get_temp_dir() + reference_dir = get_test_loc("test_util/licenses") + fields = [ + ("license_expression", "mit or public-domain"), + ("about_resource", "."), + ("name", "test"), + ("license_key", ["mit", "public-domain"]), + ("license_file", ["mit.LICENSE, mit2.LICENSE", "public-domain.LICENSE"]), + ] + util.copy_license_notice_files(fields, base_dir, reference_dir, "") + licenses = ["mit.LICENSE", "mit2.LICENSE", "public-domain.LICENSE"] + from os import listdir + + copied_files = listdir(base_dir) + assert len(licenses) == len(copied_files) + for license in licenses: + assert license in copied_files + + def test_copy_file(self): + des = get_temp_dir() + test_file = get_test_loc("test_util/licenses/mit.LICENSE") + licenses = ["mit.LICENSE"] + err = util.copy_file(test_file, des) + from os import listdir + + copied_files = listdir(des) + assert len(licenses) == len(copied_files) + assert err == "" + for license in licenses: + assert license in copied_files + + def test_copy_file_with_dir(self): + des = get_temp_dir() + test_dir = get_test_loc("test_util/licenses/") + licenses = ["mit.LICENSE", "mit2.LICENSE", "public-domain.LICENSE"] + err = util.copy_file(test_dir, des) + assert err == "" + + import os + + files_list = [] + dir_list = [] + # Get the directories and files in the 'des' recursively + for root, dir, files in os.walk(des): + for d in dir: + dir_list.append(d) + for f in files: + files_list.append(f) + + # assert dir_list == [u'licenses'] + assert len(licenses) == len(files_list) + for license in licenses: + assert license in files_list + + def test_strip_inventory_value(self): + test = [ + {"about_resource": "empty_newlines.rpm\n\n", "name": "empty_newlines.rpm"}, + {"about_resource": "spaces_after.rpm ", "name": "spaces_after.rpm "}, + {"about_resource": "value_after_newline\n123.rpm ", "name": "value_after"}, + ] + expected = [ + {"about_resource": "empty_newlines.rpm", "name": "empty_newlines.rpm"}, + {"about_resource": "spaces_after.rpm", "name": "spaces_after.rpm"}, + {"about_resource": "value_after_newline\n123.rpm", "name": "value_after"}, + ] + stripped_result = util.strip_inventory_value(test) + assert stripped_result == expected diff --git a/tests/test_validate.py b/tests/test_validate.py deleted file mode 100644 index a0a50f9b..00000000 --- a/tests/test_validate.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf8 -*- - -# ============================================================================ -# Copyright (c) 2018 nexB Inc. http://www.nexb.com/ - All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import os - -from testing_utils import run_about_command_test - -""" -Common and global checks such as codestyle and check own ABOUT files. -""" - - -root_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - - -def disabled_test_codestyle(): - - # TODO: enable me - import subprocess - args = [ - os.path.join(root_dir, 'bin', 'pycodestyle'), - '--ignore', - 'E501,W503,W504,W605', - '--exclude=lib,lib64,tests,thirdparty,docs,bin,man,settings,local,tmp', - '.', - ] - - subprocess.check_output(args=args, cwd=root_dir) - - -def test_about_thirdparty(): - run_about_command_test(['check', 'thirdparty']) - - -def test_about_src(): - run_about_command_test(['check', 'src']) - - -def test_about_etc(): - run_about_command_test(['check', 'etc']) - - -def test_about_myself(): - run_about_command_test(['check', 'about.ABOUT']) diff --git a/tests/testdata/test_attrib/default_template/expect.html b/tests/testdata/test_attrib/default_template/expect.html new file mode 100644 index 00000000..22fa81e9 --- /dev/null +++ b/tests/testdata/test_attrib/default_template/expect.html @@ -0,0 +1,67 @@ + + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + +

    cryptohash-sha256 v 0.11.100.1

    + +
    + +
    + + +
    +

    cryptohash-sha256 v 0.11.100.1

    + +

    This component is licensed under isc

    + + + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + +

    isc

    + 
     Permission to use, copy, modify, and/or distribute this software for any purpose
+with or without fee is hereby granted, provided that the above copyright notice
+and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
+THIS SOFTWARE.
+
    + + + +

    End

    + + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/default_template/simple_sample.csv b/tests/testdata/test_attrib/default_template/simple_sample.csv new file mode 100644 index 00000000..092d18b8 --- /dev/null +++ b/tests/testdata/test_attrib/default_template/simple_sample.csv @@ -0,0 +1,2 @@ +name,version,license_expression,about_resource +cryptohash-sha256,v 0.11.100.1,isc,/project/cryptohash-sha256 diff --git a/tests/testdata/test_attrib/gen_default_template/expected_default_attrib.html b/tests/testdata/test_attrib/gen_default_template/expected_default_attrib.html index 3fba0f1f..95f2ceb5 100644 --- a/tests/testdata/test_attrib/gen_default_template/expected_default_attrib.html +++ b/tests/testdata/test_attrib/gen_default_template/expected_default_attrib.html @@ -1,3 +1,4 @@ + @@ -8,43 +9,31 @@ Open Source Software Information -

    OPEN SOURCE SOFTWARE INFORMATION

    -
    -

    Licenses, acknowledgments and required copyright notices for - open source components:

    +

    +

    Licenses, acknowledgments and required copyright notices for open source components:

    +
    + +

    Apache HTTP Server 2.4.3

    +
    - -
    - -

    Apache HTTP Server 2.4.3

    - -
    - -
    - - - -
    -

    Apache HTTP Server - 2.4.3 -

    - - - - - - -
    - - -
    - -

    Common Licenses Used in This Product

    - - - -

    End

    - - +
    + +
    +

    Apache HTTP Server 2.4.3

    + + + + + + +
    + +
    +

    Common Licenses Used in This Product

    + +

    End

    + This file was generated with AboutCode Toolkit version: 6.0.0 on: 2021-08-30 10:58:38.548879 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/Apache-2.0.txt b/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/Apache-2.0.txt index 9e2c451f..71b05432 100644 --- a/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/Apache-2.0.txt +++ b/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/Apache-2.0.txt @@ -1 +1 @@ -This is Apache \ No newline at end of file +This is Apache diff --git a/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/LGPL-3.0.txt b/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/LGPL-3.0.txt index b2895e88..785b0f03 100644 --- a/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/LGPL-3.0.txt +++ b/tests/testdata/test_attrib/gen_license_key_name_check/LICENSES/LGPL-3.0.txt @@ -1 +1 @@ -This is LGPL \ No newline at end of file +This is LGPL diff --git a/tests/testdata/test_attrib/gen_license_key_name_check/expected/expected.html b/tests/testdata/test_attrib/gen_license_key_name_check/expected/expected.html index 12432aaa..7c2c3858 100644 --- a/tests/testdata/test_attrib/gen_license_key_name_check/expected/expected.html +++ b/tests/testdata/test_attrib/gen_license_key_name_check/expected/expected.html @@ -10,20 +10,22 @@ - +
    - +

    Apache-2.0

    - - 
    This is Apache
    - + + 
    This is Apache
+
    +

    LGPL-3.0-or-later

    - - 
    This is LGPL
    - - + + 
    This is LGPL
+
    + +
    - +
    diff --git a/tests/testdata/test_attrib/gen_license_key_name_check/test.ABOUT b/tests/testdata/test_attrib/gen_license_key_name_check/test.ABOUT index 166827c6..bee19473 100644 --- a/tests/testdata/test_attrib/gen_license_key_name_check/test.ABOUT +++ b/tests/testdata/test_attrib/gen_license_key_name_check/test.ABOUT @@ -1,6 +1,6 @@ about_resource: . name: test -license_expression: LGPL-3.0-or-later OR Apache-2.0 +license_expression: Apache-2.0 OR LGPL-3.0-or-later licenses: - key: Apache-2.0 name: Apache License 2.0 diff --git a/tests/testdata/test_attrib/scancode_input/sc-2-licenses.json b/tests/testdata/test_attrib/scancode_input/sc-2-licenses.json new file mode 100644 index 00000000..f97900fc --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-2-licenses.json @@ -0,0 +1,83 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "v32.0.6-2-g200fb10eb2", + "options": { + "input": ["C:\\lic\\lic.txt"], + "--json-pp": "-", + "--license": true, + "--processes": "2" + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2023-07-26T092929.619854", + "end_timestamp": "2023-07-26T092935.415721", + "output_format_version": "3.0.0", + "duration": 5.795867443084717, + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "win", + "cpu_architecture": "64", + "platform": "Windows-10-10.0.22621-SP0", + "platform_version": "10.0.22621", + "python_version": "3.9.0 (tags/v3.9.0:9cf6752, Oct 5 2020, 15:34:40) [MSC v.1927 64 bit (AMD64)]" + }, + "spdx_license_list_version": "3.21", + "files_count": 1 + } + } + ], + "license_detections": [ + { + "identifier": "mit_and__bsd_axis_nomod_or_gpl_1_0_plus-47fd20c7-6ad8-d7cb-03b7-4c40fc9f8ded", + "license_expression": "mit AND (bsd-axis-nomod OR gpl-1.0-plus)", + "detection_count": 1 + } + ], + "files": [ + { + "path": "lic.txt", + "type": "file", + "detected_license_expression": "mit AND (bsd-axis-nomod OR gpl-1.0-plus)", + "detected_license_expression_spdx": "MIT AND (LicenseRef-scancode-bsd-axis-nomod OR GPL-1.0-or-later)", + "license_detections": [ + { + "license_expression": "mit AND (bsd-axis-nomod OR gpl-1.0-plus)", + "matches": [ + { + "score": 100.0, + "start_line": 1, + "end_line": 5, + "matched_length": 161, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "mit", + "rule_identifier": "mit.LICENSE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" + }, + { + "score": 47.56, + "start_line": 7, + "end_line": 9, + "matched_length": 39, + "match_coverage": 47.56, + "matcher": "3-seq", + "license_expression": "bsd-axis-nomod OR gpl-1.0-plus", + "rule_identifier": "bsd-axis-nomod_or_gpl2.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-axis-nomod_or_gpl2.RULE" + } + ], + "identifier": "mit_and__bsd_axis_nomod_or_gpl_1_0_plus-47fd20c7-6ad8-d7cb-03b7-4c40fc9f8ded" + } + ], + "license_clues": [], + "percentage_of_license_text": 87.34, + "scan_errors": [] + } + ] +} diff --git a/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.html b/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.html new file mode 100644 index 00000000..38b3e2aa --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.html @@ -0,0 +1,113 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + + + + + +

    readme.txt

    + + + + + +
    + +
    + + + + + + +
    +

    readme.txt

    + + + + + + 
     Copyright (c) Henrik Ravn 2004
    + + + + +

    This component is licensed under unknown-license-reference

    + +

    This component is licensed under boost-1.0

    + + + + + + + + + + + + + +

    Full text of boost-1.0 is available at the end of this document.

    + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + +

    boost-1.0

    + 
     Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
    + + + +

    End

    + + This file was generated with AttributeCode version: 9.0.0 on: 2023-08-02 08:18:32.861896 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.json b/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.json new file mode 100644 index 00000000..e7d3437c --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-dup-lic-match.json @@ -0,0 +1,122 @@ +{ + "files": [ + { + "path": "samples/zlib/dotzlib/readme.txt", + "type": "file", + "name": "readme.txt", + "base_name": "readme", + "extension": ".txt", + "size": 2358, + "date": "2023-07-26", + "sha1": "b1229b826f0096808628474538cea8fec2922a9b", + "md5": "1f20f3168ee63d90de033edac2ce383c", + "sha256": "d04972a91b1563fb4b7acab4b9ff2b84e57368953cc0596d5f5ea17d97315fd0", + "mime_type": "text/plain", + "file_type": "ASCII text, with CRLF line terminators", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": "boost-1.0", + "detected_license_expression_spdx": "BSL-1.0", + "license_detections": [ + { + "license_expression": "boost-1.0", + "matches": [ + { + "score": 100.0, + "start_line": 10, + "end_line": 10, + "matched_length": 6, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "unknown-license-reference", + "rule_identifier": "unknown-license-reference_225.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/unknown-license-reference_225.RULE" + }, + { + "score": 100.0, + "start_line": 1, + "end_line": 23, + "matched_length": 211, + "match_coverage": 100.0, + "matcher": "1-hash", + "license_expression": "boost-1.0", + "rule_identifier": "boost-1.0.LICENSE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/boost-1.0.LICENSE" + } + ], + "identifier": "boost-1.0-5cff8c89-be8c-7d4a-7989-c20725314b4c", + "detection_log": ["unknown-reference-to-local-file"] + }, + { + "license_expression": "boost-1.0", + "matches": [ + { + "score": 100.0, + "start_line": 57, + "end_line": 58, + "matched_length": 32, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "boost-1.0", + "rule_identifier": "boost-1.0_21.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/boost-1.0_21.RULE" + }, + { + "score": 100.0, + "start_line": 1, + "end_line": 23, + "matched_length": 211, + "match_coverage": 100.0, + "matcher": "1-hash", + "license_expression": "boost-1.0", + "rule_identifier": "boost-1.0.LICENSE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/boost-1.0.LICENSE" + } + ], + "identifier": "boost-1.0-d2497bb7-4937-90c9-b38e-63d81d1b1d13", + "detection_log": ["unknown-reference-to-local-file"] + } + ], + "license_clues": [], + "percentage_of_license_text": 11.18, + "copyrights": [ + { + "copyright": "Copyright (c) Henrik Ravn 2004", + "start_line": 55, + "end_line": 55 + } + ], + "holders": [ + { + "holder": "Henrik Ravn", + "start_line": 55, + "end_line": 55 + } + ], + "authors": [], + "emails": [], + "urls": [ + { + "url": "http://www.boost.org/LICENSE_1_0.txt", + "start_line": 58, + "end_line": 58 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} diff --git a/tests/testdata/test_attrib/scancode_input/sc-dup-lic.html b/tests/testdata/test_attrib/scancode_input/sc-dup-lic.html new file mode 100644 index 00000000..6942e3d2 --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-dup-lic.html @@ -0,0 +1,97 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + + + + + +

    adler32.c

    + + + + + +
    + +
    + + + + + + +
    +

    adler32.c

    + + + + + + 
     Copyright (c) 1995-2011 Mark Adler
    + + + + +

    This component is licensed under zlib

    + + + + + +

    Full text of zlib is available at the end of this document.

    + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + +

    zlib

    + 
     This software is provided 'as-is', without any express or implied warranty. In no
+event will the authors be held liable for any damages arising from the use of this
+software.
+
+Permission is granted to anyone to use this software for any purpose, including
+commercial applications, and to alter it and redistribute it freely, subject to
+the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not claim that
+   you wrote the original software. If you use this software in a product, an
+   acknowledgment in the product documentation would be appreciated but is not
+   required.
+
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+3. This notice may not be removed or altered from any source distribution.
    + + + +

    End

    + + This file was generated with AttributeCode version: 9.0.0 on: 2023-08-01 23:26:57.694021 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/sc-dup-lic.json b/tests/testdata/test_attrib/scancode_input/sc-dup-lic.json new file mode 100644 index 00000000..cf6b769a --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-dup-lic.json @@ -0,0 +1,87 @@ +{ + "files": [ + { + "path": "samples/zlib/adler32.c", + "type": "file", + "name": "adler32.c", + "base_name": "adler32", + "extension": ".c", + "size": 4968, + "date": "2023-07-26", + "sha1": "0cff4808476ce0b5f6f0ebbc69ee2ab2a0eebe43", + "md5": "ae3bbb54820e1d49fb90cbba222e973f", + "sha256": "341d49ae2703037d2d10c8486f1a1ca3b65e0f10cc9e5fead6bfbbc0b34564ba", + "mime_type": "text/x-c", + "file_type": "C source, ASCII text", + "programming_language": "C", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": "zlib", + "detected_license_expression_spdx": "Zlib", + "license_detections": [ + { + "license_expression": "zlib", + "matches": [ + { + "score": 40.0, + "start_line": 3, + "end_line": 3, + "matched_length": 12, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "zlib", + "rule_identifier": "zlib_5.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/zlib_5.RULE" + }, + { + "score": 100.0, + "start_line": 6, + "end_line": 23, + "matched_length": 144, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "zlib", + "rule_identifier": "zlib_17.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/zlib_17.RULE" + } + ], + "identifier": "zlib-663c0d51-510f-fca6-b163-671ecb188ff9", + "detection_log": [ + "unknown-reference-to-local-file" + ] + } + ], + "license_clues": [], + "percentage_of_license_text": 2.06, + "copyrights": [ + { + "copyright": "Copyright (c) 1995-2011 Mark Adler", + "start_line": 2, + "end_line": 2 + } + ], + "holders": [ + { + "holder": "Mark Adler", + "start_line": 2, + "end_line": 2 + } + ], + "authors": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/sc-min_score-0.html b/tests/testdata/test_attrib/scancode_input/sc-min_score-0.html new file mode 100644 index 00000000..9472e34c --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-min_score-0.html @@ -0,0 +1,411 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + + + + + +

    lic.txt

    + + + + + +
    + +
    + + + + + + +
    +

    lic.txt

    + + + + + + +

    This component is licensed under mit

    + +

    This component is licensed under bsd-axis-nomod OR gpl-1.0-plus

    + + + + + +

    Full text of mit is available at the end of this document.

    + + + + + + + +

    bsd-axis-nomod

    + 
     Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer,
+   without modification.
+
+2. Neither the name of the copyright holders nor the names of its
+   contributors may be used to endorse or promote products derived
+   from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND ITS CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDERS OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
    + + + + + + + + + +

    Full text of gpl-1.0-plus is available at the end of this document.

    + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + + + +

    gpl-1.0-plus

    + 
     This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 1, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave,
+Cambridge, MA 02139, USA.
+
+
+                    GNU GENERAL PUBLIC LICENSE
+                     Version 1, February 1989
+
+ Copyright (C) 1989 Free Software Foundation, Inc.
+                    51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The license agreements of most software companies try to keep users
+at the mercy of those companies.  By contrast, our General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  The
+General Public License applies to the Free Software Foundation's
+software and to any other program whose authors commit to using it.
+You can use it for your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Specifically, the General Public License is designed to make
+sure that you have the freedom to give away or sell copies of free
+software, that you receive source code or can get it if you want it,
+that you can change the software or use pieces of it in new free
+programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of a such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must tell them their rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any program or other work which
+contains a notice placed by the copyright holder saying it may be
+distributed under the terms of this General Public License.  The
+"Program", below, refers to any such program or work, and a "work based
+on the Program" means either the Program or any work containing the
+Program or a portion of it, either verbatim or with modifications.  Each
+licensee is addressed as "you".
+
+  1. You may copy and distribute verbatim copies of the Program's source
+code as you receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice and
+disclaimer of warranty; keep intact all the notices that refer to this
+General Public License and to the absence of any warranty; and give any
+other recipients of the Program a copy of this General Public License
+along with the Program.  You may charge a fee for the physical act of
+transferring a copy.
+
+  2. You may modify your copy or copies of the Program or any portion of
+it, and copy and distribute such modifications under the terms of Paragraph
+1 above, provided that you also do the following:
+
+    a) cause the modified files to carry prominent notices stating that
+    you changed the files and the date of any change; and
+
+    b) cause the whole of any work that you distribute or publish, that
+    in whole or in part contains the Program or any part thereof, either
+    with or without modifications, to be licensed at no charge to all
+    third parties under the terms of this General Public License (except
+    that you may choose to grant warranty protection to some or all
+    third parties, at your option).
+
+    c) If the modified program normally reads commands interactively when
+    run, you must cause it, when started running for such interactive use
+    in the simplest and most usual way, to print or display an
+    announcement including an appropriate copyright notice and a notice
+    that there is no warranty (or else, saying that you provide a
+    warranty) and that users may redistribute the program under these
+    conditions, and telling the user how to view a copy of this General
+    Public License.
+
+    d) You may charge a fee for the physical act of transferring a
+    copy, and you may at your option offer warranty protection in
+    exchange for a fee.
+
+Mere aggregation of another independent work with the Program (or its
+derivative) on a volume of a storage or distribution medium does not bring
+the other work under the scope of these terms.
+
+
+  3. You may copy and distribute the Program (or a portion or derivative of
+it, under Paragraph 2) in object code or executable form under the terms of
+Paragraphs 1 and 2 above provided that you also do one of the following:
+
+    a) accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of
+    Paragraphs 1 and 2 above; or,
+
+    b) accompany it with a written offer, valid for at least three
+    years, to give any third party free (except for a nominal charge
+    for the cost of distribution) a complete machine-readable copy of the
+    corresponding source code, to be distributed under the terms of
+    Paragraphs 1 and 2 above; or,
+
+    c) accompany it with the information you received as to where the
+    corresponding source code may be obtained.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form alone.)
+
+Source code for a work means the preferred form of the work for making
+modifications to it.  For an executable file, complete source code means
+all the source code for all modules it contains; but, as a special
+exception, it need not include source code for modules which are standard
+libraries that accompany the operating system on which the executable
+file runs, or for standard header files or definitions files that
+accompany that operating system.
+
+  4. You may not copy, modify, sublicense, distribute or transfer the
+Program except as expressly provided under this General Public License.
+Any attempt otherwise to copy, modify, sublicense, distribute or transfer
+the Program is void, and will automatically terminate your rights to use
+the Program under this License.  However, parties who have received
+copies, or rights to use copies, from you under this General Public
+License will not have their licenses terminated so long as such parties
+remain in full compliance.
+
+  5. By copying, distributing or modifying the Program (or any work based
+on the Program) you indicate your acceptance of this license to do so,
+and all its terms and conditions.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the original
+licensor to copy, distribute or modify the Program subject to these
+terms and conditions.  You may not impose any further restrictions on the
+recipients' exercise of the rights granted herein.
+
+
+  7. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of the license which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+the license, you may choose any version ever published by the Free Software
+Foundation.
+
+  8. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  9. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  10. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+
+        Appendix: How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to humanity, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these
+terms.
+
+  To do so, attach the following notices to the program.  It is safest to
+attach them to the start of each source file to most effectively convey
+the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) 19yy  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 1, or (at your option)
+    any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
+
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) 19xx name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the
+appropriate parts of the General Public License.  Of course, the
+commands you use may be called something other than `show w' and `show
+c'; they could even be mouse-clicks or menu items--whatever suits your
+program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  program `Gnomovision' (a program to direct compilers to make passes
+  at assemblers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+That's all there is to it!
    + + + +

    mit

    + 
     Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    + + + +

    End

    + + This file was generated with AttributeCode version: 9.0.0 on: 2023-08-01 04:12:51.254562 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/sc-multi-lic.html b/tests/testdata/test_attrib/scancode_input/sc-multi-lic.html new file mode 100644 index 00000000..9bce7378 --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-multi-lic.html @@ -0,0 +1,110 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + + + + + +

    S3_PING.java

    + + + + + +
    + +
    + + + + + + +
    +

    S3_PING.java

    + + + + + + +

    This component is licensed under public-domain

    + +

    This component is licensed under public-domain-disclaimer

    + + + + + + + +

    public-domain

    +
    + + + + + + + + + + + + + +

    public-domain-disclaimer

    + 
     This code is hereby placed in the public domain.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    + + + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + + + + + +

    End

    + + This file was generated with AttributeCode version: 9.0.0 on: 2023-08-02 09:05:00.435297 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/sc-multi-lic.json b/tests/testdata/test_attrib/scancode_input/sc-multi-lic.json new file mode 100644 index 00000000..8400ac0c --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc-multi-lic.json @@ -0,0 +1,111 @@ +{ + "files": [ + { + "path": "samples/JGroups/src/S3_PING.java", + "type": "file", + "name": "S3_PING.java", + "base_name": "S3_PING", + "extension": ".java", + "size": 122528, + "date": "2023-07-26", + "sha1": "08dba9986f69719970ead3592dc565465164df0d", + "md5": "83d8324f37d0e3f120bc89865cf0bd39", + "sha256": "c4d59a8837c6320788c74496201e3ecc0ff2100525ebb727bcae6d855b34c548", + "mime_type": "text/x-java", + "file_type": "Java source, ASCII text", + "programming_language": "Java", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": false, + "package_data": [], + "for_packages": [], + "detected_license_expression": "public-domain AND public-domain-disclaimer", + "detected_license_expression_spdx": "LicenseRef-scancode-public-domain AND LicenseRef-scancode-public-domain-disclaimer", + "license_detections": [ + { + "license_expression": "public-domain", + "matches": [ + { + "score": 70.0, + "start_line": 1649, + "end_line": 1649, + "matched_length": 2, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "public-domain", + "rule_identifier": "public-domain_bare_words.RULE", + "rule_relevance": 70, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/public-domain_bare_words.RULE" + } + ], + "identifier": "public_domain-3dd945ae-65df-7d90-6467-60f8ecf2eb77" + }, + { + "license_expression": "public-domain-disclaimer", + "matches": [ + { + "score": 100.0, + "start_line": 1692, + "end_line": 1694, + "matched_length": 30, + "match_coverage": 100.0, + "matcher": "2-aho", + "license_expression": "public-domain-disclaimer", + "rule_identifier": "public-domain-disclaimer_77.RULE", + "rule_relevance": 100, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/public-domain-disclaimer_77.RULE" + } + ], + "identifier": "public_domain_disclaimer-5765d0b6-4dea-c655-1767-e7e398a296d3" + } + ], + "license_clues": [], + "percentage_of_license_text": 0.27, + "copyrights": [], + "holders": [], + "authors": [ + { + "author": "Bela Ban", + "start_line": 37, + "end_line": 37 + }, + { + "author": "Robert Harder", + "start_line": 1698, + "end_line": 1698 + }, + { + "author": "rob@iharder.net", + "start_line": 1699, + "end_line": 1699 + } + ], + "emails": [ + { + "email": "rob@iharder.net", + "start_line": 1699, + "end_line": 1699 + } + ], + "urls": [ + { + "url": "http://iharder.sourceforge.net/current/java/base64/", + "start_line": 1652, + "end_line": 1652 + }, + { + "url": "http://iharder.net/base64", + "start_line": 1695, + "end_line": 1695 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} diff --git a/tests/testdata/test_attrib/scancode_input/sc.html b/tests/testdata/test_attrib/scancode_input/sc.html new file mode 100644 index 00000000..115238d7 --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/sc.html @@ -0,0 +1,94 @@ + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + + + + + +

    lic.txt

    + + + + + +
    + +
    + + + + + + +
    +

    lic.txt

    + + + + + + +

    This component is licensed under mit

    + + + + + +

    Full text of mit is available at the end of this document.

    + + + + +
    + + +
    + +

    Common Licenses Used in This Product

    + + +

    mit

    + 
     Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    + + + +

    End

    + + This file was generated with AttributeCode version: 9.0.0 on: 2023-08-01 04:13:10.236860 (UTC) + + \ No newline at end of file diff --git a/tests/testdata/test_attrib/scancode_input/scancode.template b/tests/testdata/test_attrib/scancode_input/scancode.template new file mode 100644 index 00000000..935443ef --- /dev/null +++ b/tests/testdata/test_attrib/scancode_input/scancode.template @@ -0,0 +1,97 @@ +{# +about object and license dictionary are passed. +See https://tdruez.github.io/licenses/ +Read the JSON file to see what information can be extracted from the licenses. +#} + + + + + Open Source Software Information + + + +

    OPEN SOURCE SOFTWARE INFORMATION

    +

    {{ vartext['subtitle'] }}

    +
    +

    Licenses, acknowledgments and required copyright notices for + open source components:

    +
    + +
    + {% for about_object in abouts %} + {% set glob={'handled': True} %} + {% set index = loop.index0 %} + {% for lic in about_object.licenses if lic.score >= min_license_score %} {# license score condition #} + {% if glob['handled'] %} +

    {{ about_object.name }}{% if about_object.version %} {{ about_object.version }}{% endif %}

    + {% set _ = glob.update({'handled':false}) %} + {% endif %} + {% endfor %} + {% endfor %} +
    + +
    + + {% set all_licenses = [] %} + {% for about_object in abouts %} + {% set component_licenses = [] %} + {% set index = loop.index0 %} + {% for lic in about_object.licenses if lic.score >= min_license_score %} {# license score condition #} + {% if not lic.key in component_licenses %} + {{ component_licenses.append(lic.key)|default("", True) }} + {{ all_licenses.append(lic.key)|default("", True) }} + {% endif %} + {% endfor %} + + {% if component_licenses %} +
    +

    {{ about_object.name }} + {% if about_object.version %}{{ about_object.version }}{% endif %} +

    +

    This component is licensed under + {% for license_key in component_licenses %} + {{ license_key }} + {% endfor %} + {% if about_object.copyrights %} + {% for copyright in about_object.copyrights %} + 

    {{copyright['value']}}
    + {% endfor %} + {% endif %} + {% set glob={} %} + {% for lic in component_licenses %} + {% if not glob[lic] %} + {% if lic in license_dict %} +

    Full text of + + {{ lic }} + + is available at the end of this document.

    + {% set _ = glob.update({ lic: true }) %} + {% endif %} + {% endif %} + {% endfor %} +
    + {% endif %} + {% endfor %} + +
    + +

    Licenses Used in This Product

    + {% for key in license_dict %} + {% if key in all_licenses %} +

    {{ key }}

    + 
    {{ license_dict[key].license_text|e }}
    + 
    ({{ license_dict[key].homepage_url|e }})
    + {% endif %} + {% endfor %} + +

    End

    + This file was generated with AttributeCode version: {{ tkversion }} on: {{ utcnow }} (UTC) + + + diff --git a/tests/testdata/test_cmd/help/about_attrib_help.txt b/tests/testdata/test_cmd/help/about_attrib_help.txt index e2a0a8a6..caa7c05e 100644 --- a/tests/testdata/test_cmd/help/about_attrib_help.txt +++ b/tests/testdata/test_cmd/help/about_attrib_help.txt @@ -1,17 +1,30 @@ -Usage: about attrib [OPTIONS] LOCATION OUTPUT +Usage: about attrib [OPTIONS] INPUT OUTPUT - Generate an attribution document at OUTPUT using .ABOUT files at LOCATION. + Generate an attribution document at OUTPUT using JSON, CSV or XLSX or .ABOUT + files at INPUT. - LOCATION: Path to a file, directory or .zip archive containing .ABOUT files. + INPUT: Path to a file (.ABOUT/.csv/.json/.xlsx), directory or .zip archive + containing .ABOUT files. OUTPUT: Path where to write the attribution document. Options: - --template FILE Path to an optional custom attribution template to - generate the attribution document. If not provided - the default built-in template is used. - --vartext = Add variable text as key=value for use in a custom - attribution template. - -q, --quiet Do not print error or warning messages. - --verbose Show all error and warning messages. - -h, --help Show this message and exit. + --api_url URL URL to DejaCode License Library. + --api_key KEY API Key for the DejaCode License Library + --min-license-score INTEGER Attribute components that have license score + higher than or equal to the defined --min- + license-score. + --scancode Indicate the input JSON file is from + scancode_toolkit. + --reference DIR Path to a directory with reference files where + "license_file" and/or "notice_file" located. + --template FILE Path to an optional custom attribution template + to generate the attribution document. If not + provided the default built-in template is used. + --vartext = Add variable text as key=value for use in a + custom attribution template. + --worksheet name The worksheet name from the INPUT. (Default: the + "active" worksheet) + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/help/about_check_help.txt b/tests/testdata/test_cmd/help/about_check_help.txt index e895edc4..28f71ce4 100644 --- a/tests/testdata/test_cmd/help/about_check_help.txt +++ b/tests/testdata/test_cmd/help/about_check_help.txt @@ -2,8 +2,14 @@ Usage: about check [OPTIONS] LOCATION Check .ABOUT file(s) at LOCATION for validity and print error messages. - LOCATION: Path to a file or directory containing .ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. Options: - --verbose Show all error and warning messages. - -h, --help Show this message and exit. + --exclude PATTERN Exclude the processing of the specified input pattern + (e.g. *tests* or test/). + --license Validate the license_expression value in the input. + --djc api_url api_key Validate license_expression from a DejaCode License + Library API URL using the API KEY. + --log FILE Path to a file to save the error messages if any. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/help/about_gen_help.txt b/tests/testdata/test_cmd/help/about_gen_help.txt index 2ff956c6..40d9182d 100644 --- a/tests/testdata/test_cmd/help/about_gen_help.txt +++ b/tests/testdata/test_cmd/help/about_gen_help.txt @@ -1,17 +1,27 @@ Usage: about gen [OPTIONS] LOCATION OUTPUT - Generate .ABOUT files in OUTPUT from an inventory of .ABOUT files at - LOCATION. + Given a CSV/JSON/XLSX inventory, generate ABOUT files in the output location. - LOCATION: Path to a JSON or CSV inventory file. + LOCATION: Path to a JSON/CSV/XLSX inventory file. OUTPUT: Path to a directory where ABOUT files are generated. Options: - --fetch-license URL KEY Fetch license data and text files from a DejaCode - License Library API URL using the API KEY. - --reference DIR Path to a directory with reference license data and - text files. - -q, --quiet Do not print error or warning messages. - --verbose Show all error and warning messages. - -h, --help Show this message and exit. + --android Generate MODULE_LICENSE_XXX (XXX will be + replaced by license key) and NOTICE as the + same design as from Android. + --fetch-license Fetch license data and text files from the + ScanCode LicenseDB. + --fetch-license-djc api_url api_key + Fetch license data and text files from a + DejaCode License Library API URL using the API + KEY. + --scancode Indicate the input JSON file is from + scancode_toolkit. + --reference DIR Path to a directory with reference license + data and text files. + --worksheet name The worksheet name from the INPUT. (Default: + the "active" worksheet) + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/help/about_gen_license_help.txt b/tests/testdata/test_cmd/help/about_gen_license_help.txt new file mode 100644 index 00000000..802598f2 --- /dev/null +++ b/tests/testdata/test_cmd/help/about_gen_license_help.txt @@ -0,0 +1,16 @@ +Usage: about gen-license [OPTIONS] LOCATION OUTPUT + + Fetch licenses (Default: ScanCode LicenseDB) in the license_expression field + and save to the output location. + + LOCATION: Path to a JSON/CSV/XLSX/.ABOUT file(s) + + OUTPUT: Path to a directory where license files are saved. + +Options: + --djc api_url api_key Fetch licenses from a DejaCode License Library. + --scancode Indicate the input JSON file is from scancode_toolkit. + --worksheet name The worksheet name from the INPUT. (Default: the + "active" worksheet) + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/help/about_help.txt b/tests/testdata/test_cmd/help/about_help.txt index 173b1c44..5e672916 100644 --- a/tests/testdata/test_cmd/help/about_help.txt +++ b/tests/testdata/test_cmd/help/about_help.txt @@ -13,9 +13,15 @@ Options: -h, --help Show this message and exit. Commands: - attrib Generate an attribution document from .ABOUT files. - check Validate that the format of .ABOUT files is correct and report - errors and warnings. - gen Generate .ABOUT files from an inventory as CSV or JSON. - inventory Collect the inventory of .ABOUT files to a CSV or JSON file. - transform Transform a CSV by applying renamings, filters and checks. + attrib Generate an attribution document from JSON/CSV/XLSX/.ABOUT + files. + check Validate that the format of .ABOUT files is correct and + report errors and warnings. + collect-redist-src Collect redistributable sources. + gen Generate .ABOUT files from an inventory as CSV/JSON/XLSX. + gen-license Fetch and save all the licenses in the license_expression + field to a directory. + inventory Collect the inventory of .ABOUT files to a CSV/JSON/XLSX + file or stdout. + transform Transform a CSV/JSON/XLSX by applying renamings, filters + and checks. diff --git a/tests/testdata/test_cmd/help/about_inventory_help.txt b/tests/testdata/test_cmd/help/about_inventory_help.txt index 34fe023f..8aa41742 100644 --- a/tests/testdata/test_cmd/help/about_inventory_help.txt +++ b/tests/testdata/test_cmd/help/about_inventory_help.txt @@ -1,13 +1,18 @@ Usage: about inventory [OPTIONS] LOCATION OUTPUT - Collect the inventory of .ABOUT file data as CSV or JSON. + Collect the inventory of .ABOUT files to a CSV/JSON/XLSX file. - LOCATION: Path to an .ABOUT file or a directory with .ABOUT files. + LOCATION: Path to an ABOUT file or a directory with ABOUT files. - OUTPUT: Path to the JSON or CSV inventory file to create. + OUTPUT: Path to the CSV/JSON/XLSX inventory file to create, or using '-' to + print result on screen/to stdout (Excel-formatted output cannot be used in + stdout). Options: - -f, --format [json|csv] Set OUTPUT inventory file format. [default: csv] - -q, --quiet Do not print error or warning messages. - --verbose Show all error and warning messages. - -h, --help Show this message and exit. + --exclude PATTERN Exclude the processing of the specified input + pattern (e.g. *tests* or test/). + -f, --format [json|csv|excel] Set OUTPUT inventory file format. [default: + csv] + -q, --quiet Do not print error or warning messages. + --verbose Show all error and warning messages. + -h, --help Show this message and exit. diff --git a/tests/testdata/test_cmd/help/about_transform_config_help.txt b/tests/testdata/test_cmd/help/about_transform_config_help.txt index 118995b3..5b1561f4 100644 --- a/tests/testdata/test_cmd/help/about_transform_config_help.txt +++ b/tests/testdata/test_cmd/help/about_transform_config_help.txt @@ -5,42 +5,55 @@ format, using the same format as an .ABOUT file. The attributes that can be set in a configuration file are: -* column_renamings: -An optional map of source CSV column name to target CSV new column name that -is used to rename CSV columns. +* field_renamings: +An optional map of source CSV or JSON field name to target CSV/JSON new field name that +is used to rename CSV fields. -For instance with this configuration the columns "Directory/Location" will be +For instance with this configuration the fields "Directory/Location" will be renamed to "about_resource" and "foo" to "bar": - column_renamings: - 'Directory/Location' : about_resource - foo : bar + field_renamings: + about_resource : 'Directory/Location' + bar : foo The renaming is always applied first before other transforms and checks. All -other column names referenced below are these that exist AFTER the renamings -have been applied to the existing column names. +other field names referenced below are these that exist AFTER the renamings +have been applied to the existing field names. -* required_columns: -An optional list of required column names that must have a value, beyond the -standard columns names. If a source CSV does not have such a column or a row is -missing a value for a required column, an error is reported. +* required_fields: +An optional list of required field names that must have a value, beyond the +standard fields names. If a source CSV/JSON does not have such a field or a row is +missing a value for a required field, an error is reported. -For instance with this configuration an error will be reported if the columns +For instance with this configuration an error will be reported if the fields "name" and "version" are missing or if any row does not have a value set for -these columns: - required_columns: +these fields: + required_fields: - name - version -* column_filters: -An optional list of column names that should be kept in the transformed CSV. If -this list is provided, all the columns from the source CSV that should be kept -in the target CSV must be listed be even if they are standard or required -columns. If this list is not provided, all source CSV columns are kept in the -transformed target CSV. +* field_filters: +An optional list of field names that should be kept in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be kept +in the target CSV/JSON must be listed regardless of either standard or required +fields. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. -For instance with this configuration the target CSV will only contains the "name" -and "version" columns and no other column: - column_filters: +For instance with this configuration the target CSV/JSON will only contains the "name" +and "version" fields and no other field: + field_filters: - name - version +* exclude_fields: +An optional list of field names that should be excluded in the transformed CSV/JSON. If +this list is provided, all the fields from the source CSV/JSON that should be excluded +in the target CSV/JSON must be listed. Excluding standard or required fields will cause +an error. If this list is not provided, all source CSV/JSON fields are kept in the +transformed target CSV/JSON. + +For instance with this configuration the target CSV/JSON will not contain the "type" +and "temp" fields: + exclude_fields: + - type + - temp + diff --git a/tests/testdata/test_cmd/help/about_transform_help.txt b/tests/testdata/test_cmd/help/about_transform_help.txt index 0aad484e..ec6c6c2e 100644 --- a/tests/testdata/test_cmd/help/about_transform_help.txt +++ b/tests/testdata/test_cmd/help/about_transform_help.txt @@ -1,15 +1,17 @@ Usage: about transform [OPTIONS] LOCATION OUTPUT - Transform the CSV file at LOCATION by applying renamings, filters and checks - and write a new CSV to OUTPUT. + Transform the CSV/JSON/XLSX file at LOCATION by applying renamings, filters + and checks and then write a new CSV/JSON/XLSX to OUTPUT. - LOCATION: Path to a CSV file. + LOCATION: Path to a CSV/JSON/XLSX file. - OUTPUT: Path to CSV inventory file to create. + OUTPUT: Path to CSV/JSON/XLSX inventory file to create. Options: -c, --configuration FILE Path to an optional YAML configuration file. See --help-format for format help. + --worksheet name The worksheet name from the INPUT. (Default: the + "active" worksheet) --help-format Show configuration file format help and exit. -q, --quiet Do not print error or warning messages. --verbose Show all error and warning messages. diff --git a/tests/testdata/test_cmd/repository-mini/appdirs.ABOUT b/tests/testdata/test_cmd/repository-mini/appdirs.ABOUT index 50ef9e6d..78da7364 100644 --- a/tests/testdata/test_cmd/repository-mini/appdirs.ABOUT +++ b/tests/testdata/test_cmd/repository-mini/appdirs.ABOUT @@ -8,5 +8,4 @@ homepage_url: https://pypi.python.org/pypi/appdirs copyright: Copyright (c) 2010 ActiveState Software Inc. license_expression: mit -license_text_file: - - appdirs.LICENSE +license_file: appdirs.LICENSE \ No newline at end of file diff --git a/tests/testdata/test_gen/inv7.csv b/tests/testdata/test_gen/inv7.csv new file mode 100644 index 00000000..bb2df8a7 --- /dev/null +++ b/tests/testdata/test_gen/inv7.csv @@ -0,0 +1,2 @@ +about_resource,name,license_expression,copyright,declared_license_expression,other_license_expression +/test.c,test.c,mit,robot,isc,public-domain diff --git a/tests/testdata/test_gen/inv_no_about_resource.csv b/tests/testdata/test_gen/inv_no_about_resource.csv new file mode 100644 index 00000000..628c89a5 --- /dev/null +++ b/tests/testdata/test_gen/inv_no_about_resource.csv @@ -0,0 +1,2 @@ +name,version,license_expression +AboutCode,0.11.0,apache-2.0 diff --git a/tests/testdata/test_gen/inventory/complex/about/Jinja2.ABOUT b/tests/testdata/test_gen/inventory/complex/about/Jinja2.ABOUT index 1fff666d..a6bd5dc4 100644 --- a/tests/testdata/test_gen/inventory/complex/about/Jinja2.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/Jinja2.ABOUT @@ -5,8 +5,8 @@ download_url: https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.7.3.tar. name: Jinja2 homepage_url: http://jinja.pocoo.org/ -dje_license: bsd-new -license_text_file: Jinja2.LICENSE +license_expression: bsd-new +license_file: Jinja2.LICENSE vcs_tool: git vcs_repository: https://github.com/mitsuhiko/jinja2.git diff --git a/tests/testdata/test_gen/inventory/complex/about/MarkupSafe.ABOUT b/tests/testdata/test_gen/inventory/complex/about/MarkupSafe.ABOUT index dd631ddf..caf52719 100644 --- a/tests/testdata/test_gen/inventory/complex/about/MarkupSafe.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/MarkupSafe.ABOUT @@ -5,11 +5,11 @@ download_url: https://pypi.python.org/packages/source/m/MarkupSafe/MarkupSafe-0. name: MarkupSafe homepage_url: https://github.com/mitsuhiko/markupsafe -dje_license: bsd-new +license_expression: bsd-new vcs_tool: git vcs_repository: https://github.com/mitsuhiko/jinja2.git -license_text_file: MarkupSafe.LICENSE +license_file: MarkupSafe.LICENSE copyright: Copyright (c) 2010 by Armin Ronacher and contributors. owner: Armin Ronacher diff --git a/tests/testdata/test_gen/inventory/complex/about/certifi.ABOUT b/tests/testdata/test_gen/inventory/complex/about/certifi.ABOUT index 37c0519b..536080e8 100644 --- a/tests/testdata/test_gen/inventory/complex/about/certifi.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/certifi.ABOUT @@ -7,4 +7,4 @@ contact: me@kennethreitz.com homepage_url: http://python-requests.org name: certifi -dje_license: mpl-2.0 \ No newline at end of file +license_expression: mpl-2.0 \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/click.ABOUT b/tests/testdata/test_gen/inventory/complex/about/click.ABOUT index c38195c3..8cd7fa2c 100644 --- a/tests/testdata/test_gen/inventory/complex/about/click.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/click.ABOUT @@ -10,8 +10,8 @@ vcs_repository: https://github.com/mitsuhiko/click.git description: | A simple wrapper around optparse for powerful command line utilities. -dje_license: bsd-new -license_text_file: click.LICENSE +license_expression: bsd-new +license_file: click.LICENSE notes: | Click uses parts of optparse written by Gregory P. Ward and maintained by the Python software foundation. This is limited to code in the parser.py diff --git a/tests/testdata/test_gen/inventory/complex/about/colorama.ABOUT b/tests/testdata/test_gen/inventory/complex/about/colorama.ABOUT index 3a9a78e8..9be4a5d6 100644 --- a/tests/testdata/test_gen/inventory/complex/about/colorama.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/colorama.ABOUT @@ -5,7 +5,7 @@ description: Cross-platform colored terminal text. homepage_url: https://pypi.python.org/pypi/colorama owner: Jonathan Hartley contact: tartley@tartley.com -dje_license: bds-new +license_expression: bsd-new keywords: color colour terminal text ansi windows crossplatform xplatform -license_text_file: colorama.LICENSE +license_file: colorama.LICENSE download_url: https://pypi.python.org/packages/source/c/colorama/colorama-0.3.1.tar.gz#md5=95ce8bf32f5c25adea14b809db3509cb \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/pip.ABOUT b/tests/testdata/test_gen/inventory/complex/about/pip.ABOUT index f8b6fa21..acaa073d 100644 --- a/tests/testdata/test_gen/inventory/complex/about/pip.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/pip.ABOUT @@ -8,8 +8,8 @@ contact: python-virtualenv@groups.google.com homepage_url: http://www.pip-installer.org author_file: pip.AUTHORS -dje_license: mit, lgpl-2.1 -license_text_file: pip.LICENSE +license_expression: mit AND lgpl-2.1 +license_file: pip.LICENSE vcs_tool: git vcs_repository: https://github.com/pypa/pip.git \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/py.ABOUT b/tests/testdata/test_gen/inventory/complex/about/py.ABOUT index cdc78953..968c7c56 100644 --- a/tests/testdata/test_gen/inventory/complex/about/py.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/py.ABOUT @@ -7,8 +7,8 @@ description: library with cross-python path, ini-parsing, io, code, log faciliti homepage_url: http://pylib.readthedocs.org/ owner: holger krekel, Ronny Pfannschmidt, Benjamin Peterson and others contact: pytest-dev@python.org -dje_license: mit -license_text_file: py.LICENSE +license_expression: mit +license_file: py.LICENSE copyright: Holger Krekel and others, 2004-2014 diff --git a/tests/testdata/test_gen/inventory/complex/about/pytest.ABOUT b/tests/testdata/test_gen/inventory/complex/about/pytest.ABOUT index bc05ec02..5bbecee4 100644 --- a/tests/testdata/test_gen/inventory/complex/about/pytest.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/pytest.ABOUT @@ -7,6 +7,6 @@ description: pytest - simple powerful testing with Python homepage_url: http://pytest.org owner: Holger Krekel, Benjamin Peterson, Ronny Pfannschmidt, Floris Bruynooghe and others contact: holger at merlinux.eu -dje_license: mit -license_text_file: pytest.LICENSE +license_expression: mit +license_file: pytest.LICENSE copyright: Copyright Holger Krekel and others, 2004-2014 diff --git a/tests/testdata/test_gen/inventory/complex/about/schematics.ABOUT b/tests/testdata/test_gen/inventory/complex/about/schematics.ABOUT index 8fdaf90a..c95630c2 100644 --- a/tests/testdata/test_gen/inventory/complex/about/schematics.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/schematics.ABOUT @@ -8,5 +8,5 @@ homepage_url: https://github.com/schematics/schematics owner: J2 Labs LLC. copyright: Copyright (c) 2013, J2 Labs LLC. -dje_license: bsd-new -license_text_file: schematics.LICENSE \ No newline at end of file +license_expression: bsd-new +license_file: schematics.LICENSE \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/setuptools.ABOUT b/tests/testdata/test_gen/inventory/complex/about/setuptools.ABOUT index 80231477..bb744412 100644 --- a/tests/testdata/test_gen/inventory/complex/about/setuptools.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/setuptools.ABOUT @@ -7,5 +7,5 @@ download_url: https://pypi.python.org/packages/3.4/s/setuptools/setuptools-5.6-p homepage_url: https://pypi.python.org/pypi/setuptools owner: Python Packaging Authority -dje_license: psf -license_text_file: PSF.LICENSE \ No newline at end of file +license_expression: psf +license_file: PSF.LICENSE \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/unicodecsv.ABOUT b/tests/testdata/test_gen/inventory/complex/about/unicodecsv.ABOUT index 932c01e3..6107fbca 100644 --- a/tests/testdata/test_gen/inventory/complex/about/unicodecsv.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/unicodecsv.ABOUT @@ -6,8 +6,8 @@ name: unicodecsv homepage_url: https://github.com/jdunck/python-unicodecsv owner: Jeremy Dunck -dje_license: bsd-new -license_text_file: unicodecsv.LICENSE +license_expression: bsd-new +license_file: unicodecsv.LICENSE vcs_tool: git vcs_repository: https://github.com/jdunck/python-unicodecsv.git diff --git a/tests/testdata/test_gen/inventory/complex/about/virtualenv.ABOUT b/tests/testdata/test_gen/inventory/complex/about/virtualenv.ABOUT deleted file mode 100644 index 79170066..00000000 --- a/tests/testdata/test_gen/inventory/complex/about/virtualenv.ABOUT +++ /dev/null @@ -1,18 +0,0 @@ -about_resource: virtualenv-1.11.6-py2.py3-none-any.whl -version: 1.11.6 -download_url: https://raw.githubusercontent.com/pypa/virtualenv/1.11.6/virtualenv.py - -vcs_tool: git -vcs_repository: https://github.com/pypa/virtualenv.git - -name: virtualenv -homepage_url: http://virtualenv.org/ -owner: The virtualenv developers - -license_url: https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt -license_text_file: virtualenv.LICENSE -dje_license: mit -copyright: | - Copyright (c) 2007 Ian Bicking and Contributors - Copyright (c) 2009 Ian Bicking, The Open Planning Project - Copyright (c) 2011-2014 The virtualenv developers diff --git a/tests/testdata/test_gen/inventory/complex/about/virtualenv.LICENSE b/tests/testdata/test_gen/inventory/complex/about/virtualenv.LICENSE deleted file mode 100644 index 7e00d5d5..00000000 --- a/tests/testdata/test_gen/inventory/complex/about/virtualenv.LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2014 The virtualenv developers - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tests/testdata/test_gen/inventory/complex/about/virtualenv.py.ABOUT b/tests/testdata/test_gen/inventory/complex/about/virtualenv.py.ABOUT deleted file mode 100644 index b3192200..00000000 --- a/tests/testdata/test_gen/inventory/complex/about/virtualenv.py.ABOUT +++ /dev/null @@ -1,18 +0,0 @@ -about_resource: virtualenv.py -version: 1.11.6 -download_url: https://raw.githubusercontent.com/pypa/virtualenv/1.11.6/virtualenv.py - -vcs_tool: git -vcs_repository: https://github.com/pypa/virtualenv.git - -name: virtualenv -homepage_url: http://virtualenv.org/ -owner: The virtualenv developers - -license_url: https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt -license_text_file: virtualenv.LICENSE -dje_license: mit -copyright: | - Copyright (c) 2007 Ian Bicking and Contributors - Copyright (c) 2009 Ian Bicking, The Open Planning Project - Copyright (c) 2011-2014 The virtualenv developers diff --git a/tests/testdata/test_gen/inventory/complex/about/wheel.ABOUT b/tests/testdata/test_gen/inventory/complex/about/wheel.ABOUT index 338c37da..0ac2fec2 100644 --- a/tests/testdata/test_gen/inventory/complex/about/wheel.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/wheel.ABOUT @@ -10,5 +10,5 @@ vcs_repository: https://bitbucket.org/pypa/wheel copyright: | copyright (c) 2012-2014 Daniel Holth and contributors. -dje_license: mit -license_text_file: wheel.LICENSE +license_expression: mit +license_file: wheel.LICENSE diff --git a/tests/testdata/test_gen/inventory/complex/about/wincertstore.ABOUT b/tests/testdata/test_gen/inventory/complex/about/wincertstore.ABOUT index 634c43ce..bd062f06 100644 --- a/tests/testdata/test_gen/inventory/complex/about/wincertstore.ABOUT +++ b/tests/testdata/test_gen/inventory/complex/about/wincertstore.ABOUT @@ -4,8 +4,8 @@ download_url: https://pypi.python.org/packages/source/w/wincertstore/wincertstor name: wincertstore -dje_license: psf -license_text_file: wincertstore.LICENSE +license_expression: psf +license_file: wincertstore.LICENSE contact: christian@python.org owner: Christian Heimes diff --git a/tests/testdata/test_gen/inventory/complex/expected.csv b/tests/testdata/test_gen/inventory/complex/expected.csv index fda2b4de..e6d86178 100644 --- a/tests/testdata/test_gen/inventory/complex/expected.csv +++ b/tests/testdata/test_gen/inventory/complex/expected.csv @@ -1,4 +1,4 @@ -about_resource,name,version,download_url,description,homepage_url,notes,license_key,license_expression,license_name,license_file,license_url,copyright,notice_file,notice_url,redistribute,attribute,track_changes,modified,internal_use_only,changelog_file,owner,owner_url,contact,author,vcs_tool,vcs_repository,vcs_path,vcs_tag,vcs_branch,vcs_revision,checksum_md5,checksum_sha1,spec_version,author_file,dje_license,keywords,license_text_file +about_resource,name,version,download_url,description,homepage_url,notes,license_key,license_expression,license_name,license_file,license_url,copyright,notice_file,notice_url,redistribute,attribute,track_changes,modified,internal_use_only,changelog_file,owner,owner_url,contact,author,vcs_tool,vcs_repository,vcs_path,vcs_tag,vcs_branch,vcs_revision,checksum_md5,checksum_sha1,spec_version,author_file,license_expression,keywords,license_file /about/,AboutCode,0.11.0,,"AboutCode is a tool to process ABOUT files. An ABOUT file is a file.",http://dejacode.org,,apache-2.0,apache-2.0,,apache-2.0.LICENSE,,Copyright (c) 2013-2014 nexB Inc.,NOTICE,,,,,,,,nexB Inc.,,,"Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez",git,https://github.com/dejacode/about-code-tool.git,,,,,,,,,,, @@ -16,12 +16,6 @@ module and is under the same license as clikc itself.",,,,,,,,,,,,,,,Armin Ronac /about/schematics-0.9_5-py2-none-any.whl,schematics,0.9-5,https://pypi.python.org/packages/source/s/schematics/schematics-0.9-5.tar.gz#md5=82ba0d67aa2600421877edcd9e7500f7,,https://github.com/schematics/schematics,,,,,,,"Copyright (c) 2013, J2 Labs LLC.",,,,,,,,,J2 Labs LLC.,,,,,,,,,,,,,,bsd-new,,schematics.LICENSE /about/setuptools-5.6-py2.py3-none-any.whl,setuptools,5.6,https://pypi.python.org/packages/3.4/s/setuptools/setuptools-5.6-py2.py3-none-any.whl#md5=4503e42d67edc51e293ba9be4af799a5,,https://pypi.python.org/pypi/setuptools,,,,,,,,,,,,,,,,Python Packaging Authority,,,,,,,,,,,,,,psf,,PSF.LICENSE /about/unicodecsv-0.9.4-py2-none-any.whl,unicodecsv,0.9.4,https://pypi.python.org/packages/source/u/unicodecsv/unicodecsv-0.9.4.tar.gz#md5=344fa55f299ba198cb73db48546002fd,,https://github.com/jdunck/python-unicodecsv,,,,,,,,,,,,,,,,Jeremy Dunck,,,,git,https://github.com/jdunck/python-unicodecsv.git,,,,,,,,,bsd-new,,unicodecsv.LICENSE -/about/virtualenv-1.11.6-py2.py3-none-any.whl,virtualenv,1.11.6,https://raw.githubusercontent.com/pypa/virtualenv/1.11.6/virtualenv.py,,http://virtualenv.org/,,,,,,https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt,"Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2014 The virtualenv developers",,,,,,,,,The virtualenv developers,,,,git,https://github.com/pypa/virtualenv.git,,,,,,,,,mit,,virtualenv.LICENSE -/about/virtualenv.py,virtualenv,1.11.6,https://raw.githubusercontent.com/pypa/virtualenv/1.11.6/virtualenv.py,,http://virtualenv.org/,,,,,,https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt,"Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2014 The virtualenv developers",,,,,,,,,The virtualenv developers,,,,git,https://github.com/pypa/virtualenv.git,,,,,,,,,mit,,virtualenv.LICENSE /about/wheel-0.24.0-py2.py3-none-any.whl,wheel,0.24.0,https://pypi.python.org/packages/py2.py3/w/wheel/wheel-0.24.0-py2.py3-none-any.whl#md5=4c24453cda2177fd42c5d62d6434679a,,https://bitbucket.org/pypa/wheel,,,,,,,"copyright (c) 2012-2014 Daniel Holth and contributors.",,,,,,,,,,,,,hg,https://bitbucket.org/pypa/wheel,,,,,,,,,mit,,wheel.LICENSE /about/wincertstore-0.2-py2.py3-none-any.whl,wincertstore,0.2,https://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.2.zip,Python module to extract CA and CRL certs from Windows' cert store (ctypes based).,https://bitbucket.org/tiran/wincertstore,,,,,,,,,,,,,,,,Christian Heimes,,christian@python.org,,,,,,,,,,,,psf,,wincertstore.LICENSE diff --git a/tests/testdata/test_gen/lic_issue_450/custom.txt b/tests/testdata/test_gen/lic_issue_450/custom.txt new file mode 100644 index 00000000..d4eac62e --- /dev/null +++ b/tests/testdata/test_gen/lic_issue_450/custom.txt @@ -0,0 +1 @@ +This is a custom license. diff --git a/tests/testdata/test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv b/tests/testdata/test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv new file mode 100644 index 00000000..4b2f2e55 --- /dev/null +++ b/tests/testdata/test_gen/lic_issue_450/custom_and_valid_lic_key_with_file.csv @@ -0,0 +1,3 @@ +about_resource,name,license_expression,license_file +test.c,test.c,mit AND custom,custom.txt +test.h,test.h,custom AND mit,custom.txt diff --git a/tests/testdata/test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv b/tests/testdata/test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv new file mode 100644 index 00000000..16f113e9 --- /dev/null +++ b/tests/testdata/test_gen/lic_key_custom_lic_file/lic_key_with_custom_lic_file.csv @@ -0,0 +1,2 @@ +about_resource,name,license_expression,license_file +test.c,test.c,custom,custom.txt diff --git a/tests/testdata/test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv b/tests/testdata/test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv new file mode 100644 index 00000000..d36c6304 --- /dev/null +++ b/tests/testdata/test_gen/lic_key_custom_lic_file/no_lic_key_with_custom_lic_file.csv @@ -0,0 +1,2 @@ +about_resource,name,license_expression,license_file +test.c,test.c,,custom.txt diff --git a/tests/testdata/test_gen/load/clean-text-0.3.0-lceupi.json b/tests/testdata/test_gen/load/clean-text-0.3.0-lceupi.json new file mode 100644 index 00000000..048fea41 --- /dev/null +++ b/tests/testdata/test_gen/load/clean-text-0.3.0-lceupi.json @@ -0,0 +1,1603 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.2.1rc2", + "options": { + "input": [ + "C:\\Users\\XYZ\\Downloads\\clean-text-0.3.0" + ], + "--copyright": true, + "--email": true, + "--info": true, + "--json-pp": "C:\\Users\\XYZ\\Downloads\\clean-text-0.3.0-lceupi.json", + "--license": true, + "--package": true, + "--processes": "4", + "--url": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2020-12-04T093439.242970", + "end_timestamp": "2020-12-04T093541.496173", + "duration": 62.25320363044739, + "message": null, + "errors": [], + "extra_data": { + "files_count": 9 + } + } + ], + "files": [ + { + "path": "clean-text-0.3.0", + "type": "directory", + "name": "clean-text-0.3.0", + "base_name": "clean-text-0.3.0", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 9, + "dirs_count": 1, + "size_count": 32826, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/LICENSE", + "type": "file", + "name": "LICENSE", + "base_name": "LICENSE", + "extension": "", + "size": 976, + "date": "2020-12-04", + "sha1": "0ab588250797da53b428be01d813339db8d5c236", + "md5": "6f4dbb7d66bda0d1ae83b99fee451959", + "sha256": "1635765fe1ed3ff0b0af49821bd2a2a42a0b5e7334c9d644962f70274b7d4cf7", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [ + { + "value": "Copyright 2016 Chartbeat, Inc.", + "start_line": 2, + "end_line": 2 + } + ], + "holders": [ + { + "value": "Chartbeat, Inc.", + "start_line": 2, + "end_line": 2 + } + ], + "authors": [], + "packages": [], + "emails": [], + "urls": [ + { + "url": "https://github.com/chartbeat-labs/textacy", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://github.com/jfilter/clean-text", + "start_line": 8, + "end_line": 8 + }, + { + "url": "http://www.apache.org/licenses/LICENSE-2.0", + "start_line": 15, + "end_line": 15 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/PKG-INFO", + "type": "file", + "name": "PKG-INFO", + "base_name": "PKG-INFO", + "extension": "", + "size": 5791, + "date": "2020-12-04", + "sha1": "c2533b165a04b7ca076c8ac0f90be7cf49611428", + "md5": "594cd2f83e551fdeba703e18c823a3d7", + "sha256": "ac63571168b11b93951f8f5265df2807e08757e0c283a3e1aae26502bf78b45a", + "mime_type": "text/x-script.python", + "file_type": "Python script, UTF-8 Unicode text executable, with very long lines", + "programming_language": "Objective-C", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + }, + { + "key": "apache-2.0", + "score": 99.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 10, + "end_line": 10, + "matched_rule": { + "identifier": "pypi_apache_software_license.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 6, + "matched_length": 6, + "match_coverage": 100.0, + "rule_relevance": 99.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 17, + "end_line": 17, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 20, + "end_line": 20, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 46, + "end_line": 46, + "matched_rule": { + "identifier": "gpl-1.0-plus_255.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": true, + "is_license_reference": false, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 49, + "end_line": 49, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 52, + "end_line": 52, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "apache-2.0", + "score": 90.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 125, + "end_line": 127, + "matched_rule": { + "identifier": "apache-2.0_161.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 90.0 + } + } + ], + "license_expressions": [ + "apache-2.0", + "apache-2.0", + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "apache-2.0" + ], + "percentage_of_license_text": 2.38, + "copyrights": [], + "holders": [], + "authors": [ + { + "value": "Johannes Filter Author-email hi@jfilter.de", + "start_line": 7, + "end_line": 10 + } + ], + "packages": [ + { + "type": "pypi", + "namespace": null, + "name": "clean-text", + "version": "0.3.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "# `clean-text` [![Build Status](https://travis-ci.com/jfilter/clean-text.svg?branch=master)](https://travis-ci.com/jfilter/clean-text) [![PyPI](https://img.shields.io/pypi/v/clean-text.svg)](https://pypi.org/project/clean-text/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/clean-text.svg)](https://pypi.org/project/clean-text/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/clean-text)](https://pypistats.org/packages/clean-text)\n\nUser-generated content on the Web and in social media is often dirty. Preprocess your scraped data with `clean-text` to create a normalized text representation. For instance, turn this corrupted input:\n\n```txt\nA bunch of \\\\u2018new\\\\u2019 references, including [Moana](https://en.wikipedia.org/wiki/Moana_%282016_film%29).\n\n\n\u00c2\u00bbY\u00c3\u00b3\u00c3\u00b9 \u00c3\u00a0r\u00c3\u00a9 r\u00c3\u00afght <3!\u00c2\u00ab\n```\n\ninto this clean output:\n\n```txt\nA bunch of 'new' references, including [moana]().\n\n\"you are right <3!\"\n```\n\n`clean-text` uses [ftfy](https://github.com/LuminosoInsight/python-ftfy), [unidecode](https://github.com/takluyver/Unidecode) and numerous hand-crafted rules, i.e., RegEx.\n\n## Installation\n\nTo install the GPL-licensed package [unidecode](https://github.com/takluyver/Unidecode) alongside:\n\n```bash\npip install clean-text[gpl]\n```\n\nYou may want to abstain from GPL:\n\n```bash\npip install clean-text\n```\n\nNB: This package is named `clean-text` and not `cleantext`.\n\nIf [unidecode](https://github.com/takluyver/Unidecode) is not available, `clean-text` will resort to Python's [unicodedata.normalize](https://docs.python.org/3.7/library/unicodedata.html#unicodedata.normalize) for [transliteration](https://en.wikipedia.org/wiki/Transliteration).\nTransliteration to closest ASCII symbols involes manually mappings, i.e., `\u00c3\u00aa` to `e`.\n`unidecode`'s mapping is superiour but unicodedata's are sufficent.\nHowever, you may want to disable this feature altogether depending on your data and use case.\n\nTo make it clear: There are **inconsistencies** between processing text with or without `unidecode`.\n\n## Usage\n\n```python\nfrom cleantext import clean\n\nclean(\"some input\",\n fix_unicode=True, # fix various unicode errors\n to_ascii=True, # transliterate to closest ASCII representation\n lower=True, # lowercase text\n no_line_breaks=False, # fully strip line breaks as opposed to only normalizing them\n no_urls=False, # replace all URLs with a special token\n no_emails=False, # replace all email addresses with a special token\n no_phone_numbers=False, # replace all phone numbers with a special token\n no_numbers=False, # replace all numbers with a special token\n no_digits=False, # replace all digits with a special token\n no_currency_symbols=False, # replace all currency symbols with a special token\n no_punct=False, # remove punctuations\n replace_with_punct=\"\", # instead of removing punctuations you may replace them\n replace_with_url=\"\",\n replace_with_email=\"\",\n replace_with_phone_number=\"\",\n replace_with_number=\"\",\n replace_with_digit=\"0\",\n replace_with_currency_symbol=\"\",\n lang=\"en\" # set to 'de' for German special handling\n)\n```\n\nCarefully choose the arguments that fit your task. The default parameters are listed above.\n\nYou may also only use specific functions for cleaning. For this, take a look at the [source code](https://github.com/jfilter/clean-text/blob/master/cleantext/clean.py).\n\nSo far, only English and German are fully supported. It should work for the majority of western languages. If you need some special handling for your language, feel free to contribute. \u00f0\u0178\u2122\u0192\n\n## Development\n\n[Install and use poetry](https://python-poetry.org/).\n\n## Contributing\n\nIf you have a **question**, found a **bug** or want to propose a new **feature**, have a look at the [issues page](https://github.com/jfilter/clean-text/issues).\n\n**Pull requests** are especially welcomed when they fix bugs or improve the code quality.\n\nIf you don't like the output of `clean-text`, consider adding a [test](https://github.com/jfilter/clean-text/tree/master/tests) with your specific input and desired output.\n\n## Related Work\n\n- https://github.com/pudo/normality\n- https://github.com/davidmogar/cucco\n- https://github.com/lyeoni/prenlp\n- https://github.com/chartbeat-labs/textacy\n- https://github.com/jbesomi/texthero\n\n## Acknowledgements\n\nBuilt upon the work by [Burton DeWilde](https://github.com/bdewilde) for [Textacy](https://github.com/chartbeat-labs/textacy).\n\n## License\n\nApache\n\n## Sponsoring\n\nThis work was created as part of a [project](https://github.com/jfilter/ptf-kommentare) that was funded by the German [Federal Ministry of Education and Research](https://www.bmbf.de/en/index.html).\n\n\n\n", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Johannes Filter", + "email": "hi@jfilter.de", + "url": null + } + ], + "keywords": [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "apache-2.0", + "declared_license": { + "license": "Apache-2.0", + "classifiers": [ + "License :: OSI Approved :: Apache Software License" + ] + }, + "notice_text": null, + "root_path": "clean-text-0.3.0/PKG-INFO", + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": "pkg:pypi/clean-text@0.3.0", + "repository_homepage_url": "https://pypi.org/project/clean-text", + "repository_download_url": "https://pypi.io/packages/source/c/clean-text/clean-text-0.3.0.tar.gz", + "api_data_url": "http://pypi.python.org/pypi/clean-text/0.3.0/json" + } + ], + "emails": [ + { + "email": "hi@jfilter.de", + "start_line": 8, + "end_line": 8 + } + ], + "urls": [ + { + "url": "https://travis-ci.com/jfilter/clean-text.svg?branch=master", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://travis-ci.com/jfilter/clean-text", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://img.shields.io/pypi/v/clean-text.svg", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://pypi.org/project/clean-text", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://img.shields.io/pypi/pyversions/clean-text.svg", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://img.shields.io/pypi/dm/clean-text", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://pypistats.org/packages/clean-text", + "start_line": 23, + "end_line": 23 + }, + { + "url": "https://en.wikipedia.org/wiki/Moana_(2016_film)", + "start_line": 28, + "end_line": 28 + }, + { + "url": "https://github.com/LuminosoInsight/python-ftfy", + "start_line": 42, + "end_line": 42 + }, + { + "url": "https://github.com/takluyver/Unidecode", + "start_line": 42, + "end_line": 42 + }, + { + "url": "https://docs.python.org/3.7/library/unicodedata.html#unicodedata.normalize", + "start_line": 60, + "end_line": 60 + }, + { + "url": "https://en.wikipedia.org/wiki/Transliteration", + "start_line": 60, + "end_line": 60 + }, + { + "url": "https://github.com/jfilter/clean-text/blob/master/cleantext/clean.py", + "start_line": 97, + "end_line": 97 + }, + { + "url": "https://python-poetry.org/", + "start_line": 103, + "end_line": 103 + }, + { + "url": "https://github.com/jfilter/clean-text/issues", + "start_line": 107, + "end_line": 107 + }, + { + "url": "https://github.com/jfilter/clean-text/tree/master/tests", + "start_line": 111, + "end_line": 111 + }, + { + "url": "https://github.com/pudo/normality", + "start_line": 115, + "end_line": 115 + }, + { + "url": "https://github.com/davidmogar/cucco", + "start_line": 116, + "end_line": 116 + }, + { + "url": "https://github.com/lyeoni/prenlp", + "start_line": 117, + "end_line": 117 + }, + { + "url": "https://github.com/chartbeat-labs/textacy", + "start_line": 118, + "end_line": 118 + }, + { + "url": "https://github.com/jbesomi/texthero", + "start_line": 119, + "end_line": 119 + }, + { + "url": "https://github.com/bdewilde", + "start_line": 123, + "end_line": 123 + }, + { + "url": "https://github.com/jfilter/ptf-kommentare", + "start_line": 131, + "end_line": 131 + }, + { + "url": "https://www.bmbf.de/en/index.html", + "start_line": 131, + "end_line": 131 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/pyproject.toml", + "type": "file", + "name": "pyproject.toml", + "base_name": "pyproject", + "extension": ".toml", + "size": 1078, + "date": "2020-12-04", + "sha1": "39007a1ef541a36bb9f6118c0f8d6885d25b641b", + "md5": "578cdf84be7f98156c52bca8bee126cc", + "sha256": "24ccecd53315fbf5fc786ac1428791156fa91fb055f3d5e510fffae11b9a95c2", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [ + { + "key": "apache-2.0", + "score": 100.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 5, + "end_line": 5, + "matched_rule": { + "identifier": "apache-2.0_65.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 4, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + }, + { + "key": "apache-2.0", + "score": 99.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 14, + "end_line": 14, + "matched_rule": { + "identifier": "pypi_apache_software_license.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 6, + "matched_length": 6, + "match_coverage": 100.0, + "rule_relevance": 99.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 35, + "end_line": 35, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + } + ], + "license_expressions": [ + "apache-2.0", + "apache-2.0", + "gpl-1.0-plus" + ], + "percentage_of_license_text": 8.73, + "copyrights": [], + "holders": [], + "authors": [ + { + "value": "Johannes Filter ", + "start_line": 6, + "end_line": 8 + } + ], + "packages": [], + "emails": [ + { + "email": "hi@jfilter.de", + "start_line": 6, + "end_line": 6 + } + ], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/README.md", + "type": "file", + "name": "README.md", + "base_name": "README", + "extension": ".md", + "size": 4941, + "date": "2020-12-04", + "sha1": "270d7c1d78022d10fdeba2259d891746b6c41a43", + "md5": "e7b35ff1470cd41118e7c6deedd13cf1", + "sha256": "070c675a0ba7b7f7db022ce6b15c54d7a537cbeeafd414905705c2734bf9556c", + "mime_type": "text/x-script.python", + "file_type": "Python script, UTF-8 Unicode text executable, with very long lines", + "programming_language": "Objective-C", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": true, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 24, + "end_line": 24, + "matched_rule": { + "identifier": "gpl-1.0-plus_255.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": true, + "is_license_reference": false, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 27, + "end_line": 27, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 30, + "end_line": 30, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "apache-2.0", + "score": 90.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 103, + "end_line": 105, + "matched_rule": { + "identifier": "apache-2.0_161.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 90.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "apache-2.0" + ], + "percentage_of_license_text": 1.03, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [ + { + "url": "https://travis-ci.com/jfilter/clean-text.svg?branch=master", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://travis-ci.com/jfilter/clean-text", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://img.shields.io/pypi/v/clean-text.svg", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://pypi.org/project/clean-text", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://img.shields.io/pypi/pyversions/clean-text.svg", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://img.shields.io/pypi/dm/clean-text", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://pypistats.org/packages/clean-text", + "start_line": 1, + "end_line": 1 + }, + { + "url": "https://en.wikipedia.org/wiki/Moana_(2016_film)", + "start_line": 6, + "end_line": 6 + }, + { + "url": "https://github.com/LuminosoInsight/python-ftfy", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/takluyver/Unidecode", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://docs.python.org/3.7/library/unicodedata.html#unicodedata.normalize", + "start_line": 38, + "end_line": 38 + }, + { + "url": "https://en.wikipedia.org/wiki/Transliteration", + "start_line": 38, + "end_line": 38 + }, + { + "url": "https://github.com/jfilter/clean-text/blob/master/cleantext/clean.py", + "start_line": 75, + "end_line": 75 + }, + { + "url": "https://python-poetry.org/", + "start_line": 81, + "end_line": 81 + }, + { + "url": "https://github.com/jfilter/clean-text/issues", + "start_line": 85, + "end_line": 85 + }, + { + "url": "https://github.com/jfilter/clean-text/tree/master/tests", + "start_line": 89, + "end_line": 89 + }, + { + "url": "https://github.com/pudo/normality", + "start_line": 93, + "end_line": 93 + }, + { + "url": "https://github.com/davidmogar/cucco", + "start_line": 94, + "end_line": 94 + }, + { + "url": "https://github.com/lyeoni/prenlp", + "start_line": 95, + "end_line": 95 + }, + { + "url": "https://github.com/chartbeat-labs/textacy", + "start_line": 96, + "end_line": 96 + }, + { + "url": "https://github.com/jbesomi/texthero", + "start_line": 97, + "end_line": 97 + }, + { + "url": "https://github.com/bdewilde", + "start_line": 101, + "end_line": 101 + }, + { + "url": "https://github.com/jfilter/ptf-kommentare", + "start_line": 109, + "end_line": 109 + }, + { + "url": "https://www.bmbf.de/en/index.html", + "start_line": 109, + "end_line": 109 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/setup.py", + "type": "file", + "name": "setup.py", + "base_name": "setup", + "extension": ".py", + "size": 5783, + "date": "2020-12-04", + "sha1": "54e07aafcf0964a46e7ed590d539f6d54d04789d", + "md5": "1f798b06ca13803c93e67572eee2527f", + "sha256": "442944712f502093e257222c2f947576ddc6b49347a0ee1baa9ddb35ecc2f9d5", + "mime_type": "text/x-script.python", + "file_type": "Python script, UTF-8 Unicode text executable, with very long lines", + "programming_language": "Python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 14, + "end_line": 14, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 20, + "end_line": 20, + "matched_rule": { + "identifier": "gpl-1.0-plus_255.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": true, + "is_license_reference": false, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 20, + "end_line": 20, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "gpl-1.0-plus", + "score": 50.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 20, + "end_line": 20, + "matched_rule": { + "identifier": "gpl_bare_word_only.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": true, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 1, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 50.0 + } + }, + { + "key": "apache-2.0", + "score": 90.0, + "name": "Apache License 2.0", + "short_name": "Apache 2.0", + "category": "Permissive", + "is_exception": false, + "owner": "Apache Software Foundation", + "homepage_url": "http://www.apache.org/licenses/", + "text_url": "http://www.apache.org/licenses/LICENSE-2.0", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0", + "spdx_license_key": "Apache-2.0", + "spdx_url": "https://spdx.org/licenses/Apache-2.0", + "start_line": 20, + "end_line": 20, + "matched_rule": { + "identifier": "apache-2.0_161.RULE", + "license_expression": "apache-2.0", + "licenses": [ + "apache-2.0" + ], + "is_license_text": false, + "is_license_notice": false, + "is_license_reference": false, + "is_license_tag": true, + "matcher": "2-aho", + "rule_length": 2, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 90.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "gpl-1.0-plus", + "apache-2.0" + ], + "percentage_of_license_text": 1.05, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [ + { + "email": "hi@jfilter.de", + "start_line": 22, + "end_line": 22 + } + ], + "urls": [ + { + "url": "https://travis-ci.com/jfilter/clean-text.svg?branch=master", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://travis-ci.com/jfilter/clean-text", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://img.shields.io/pypi/v/clean-text.svg", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://pypi.org/project/clean-text", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://img.shields.io/pypi/pyversions/clean-text.svg", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://img.shields.io/pypi/dm/clean-text", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://pypistats.org/packages/clean-text", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://en.wikipedia.org/wiki/Moana_(2016_film)", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/LuminosoInsight/python-ftfy", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/takluyver/Unidecode", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://docs.python.org/3.7/library/unicodedata.html#unicodedata.normalize", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://en.wikipedia.org/wiki/Transliteration", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/jfilter/clean-text/blob/master/cleantext/clean.py", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://python-poetry.org/", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/jfilter/clean-text/issues", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/jfilter/clean-text/tree/master/tests", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/pudo/normality", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/davidmogar/cucco", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/lyeoni/prenlp", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/chartbeat-labs/textacy", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/jbesomi/texthero", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/bdewilde", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/jfilter/ptf-kommentare", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://www.bmbf.de/en/index.html", + "start_line": 20, + "end_line": 20 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/cleantext", + "type": "directory", + "name": "cleantext", + "base_name": "cleantext", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "sha256": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 4, + "dirs_count": 0, + "size_count": 14257, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/cleantext/__init__.py", + "type": "file", + "name": "__init__.py", + "base_name": "__init__", + "extension": ".py", + "size": 44, + "date": "2020-12-04", + "sha1": "5b59015815a0960cab67d63f9e0fd41ce800e4b9", + "md5": "c4e0e330b3f5dc1a906400d1b58d14eb", + "sha256": "14e1cf97b3e36d38bd88ebd4519693be2910e26839d29a0bc8cd1a8a7ebaec3b", + "mime_type": "text/x-script.python", + "file_type": "Python script, ASCII text executable", + "programming_language": "Python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/cleantext/clean.py", + "type": "file", + "name": "clean.py", + "base_name": "clean", + "extension": ".py", + "size": 9593, + "date": "2020-12-04", + "sha1": "ba25c99004d422e98c7f948ce6a7cf7914c69b23", + "md5": "b64befb0e9457e941e362bbb2955e5e2", + "sha256": "a089501312bc9caed493dd7cdb76f66757d92bab45c9b5861c627e2a20887573", + "mime_type": "text/plain", + "file_type": "Python script, UTF-8 Unicode text executable", + "programming_language": "Python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [ + { + "key": "gpl-1.0-plus", + "score": 100.0, + "name": "GNU General Public License 1.0 or later", + "short_name": "GPL 1.0 or later", + "category": "Copyleft", + "is_exception": false, + "owner": "Free Software Foundation (FSF)", + "homepage_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "text_url": "http://www.gnu.org/licenses/old-licenses/gpl-1.0-standalone.html", + "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:gpl-1.0-plus", + "spdx_license_key": "GPL-1.0-or-later", + "spdx_url": "https://spdx.org/licenses/GPL-1.0-or-later", + "start_line": 28, + "end_line": 28, + "matched_rule": { + "identifier": "gpl-1.0-plus_255.RULE", + "license_expression": "gpl-1.0-plus", + "licenses": [ + "gpl-1.0-plus" + ], + "is_license_text": false, + "is_license_notice": true, + "is_license_reference": false, + "is_license_tag": false, + "matcher": "2-aho", + "rule_length": 3, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100.0 + } + } + ], + "license_expressions": [ + "gpl-1.0-plus" + ], + "percentage_of_license_text": 0.24, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [ + { + "url": "http://ftfy.readthedocs.org/", + "start_line": 43, + "end_line": 43 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/cleantext/constants.py", + "type": "file", + "name": "constants.py", + "base_name": "constants", + "extension": ".py", + "size": 3590, + "date": "2020-12-04", + "sha1": "264bd4af800de52b6a84ea7f18fc30d86a5327d1", + "md5": "796b8e4292835bc239d7f97f5c556387", + "sha256": "0f346ace474b100f055ed20e99b2a1fb84e9f701429b6e00bed622b66349b5e6", + "mime_type": "text/plain", + "file_type": "Python script, UTF-8 Unicode text executable", + "programming_language": "Python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [ + { + "url": "https://github.com/jfilter/clean-text/issues/10", + "start_line": 46, + "end_line": 46 + }, + { + "url": "https://gist.github.com/dperini/729294", + "start_line": 59, + "end_line": 59 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + }, + { + "path": "clean-text-0.3.0/cleantext/specials.py", + "type": "file", + "name": "specials.py", + "base_name": "specials", + "extension": ".py", + "size": 1030, + "date": "2020-12-04", + "sha1": "3fb58ec98406168f57b34a25fc66c3eb9bda8ee1", + "md5": "966712a27803ab3a76fc140d830ecc13", + "sha256": "b6a3bb901351bceac8f854d0cc40b53af6532bd3c56a8a25de23f6fb86765265", + "mime_type": "text/x-script.python", + "file_type": "Python script, UTF-8 Unicode text executable", + "programming_language": "Python", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": true, + "is_script": true, + "licenses": [], + "license_expressions": [], + "percentage_of_license_text": 0, + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} diff --git a/tests/testdata/test_gen/load/simple_sample.xlsx b/tests/testdata/test_gen/load/simple_sample.xlsx new file mode 100644 index 00000000..4ad73e22 Binary files /dev/null and b/tests/testdata/test_gen/load/simple_sample.xlsx differ diff --git a/tests/testdata/test_gen/multi_lic_issue_443/test.csv b/tests/testdata/test_gen/multi_lic_issue_443/test.csv new file mode 100644 index 00000000..fe2b82dd --- /dev/null +++ b/tests/testdata/test_gen/multi_lic_issue_443/test.csv @@ -0,0 +1,6 @@ +about_resource,name,version,license_key,license_file +3rdParty/test,test,1.5,"License1 +License2 +License3","LIC1.LICENSE +LIC2.LICENSE +LIC3.LICENSE" diff --git a/tests/testdata/test_gen/multi_lic_issue_444/test1.csv b/tests/testdata/test_gen/multi_lic_issue_444/test1.csv new file mode 100644 index 00000000..b82173e8 --- /dev/null +++ b/tests/testdata/test_gen/multi_lic_issue_444/test1.csv @@ -0,0 +1,2 @@ +about_resource,name,license_key,license_file +3rdParty/test.c,test.c,License1,"LIC1.LICENSE, LIC2.LICENSE" diff --git a/tests/testdata/test_gen/parser_tests/about_resource_field_present.ABOUT b/tests/testdata/test_gen/parser_tests/about_resource_field_present.ABOUT index dd8d6bff..9f1d2286 100644 --- a/tests/testdata/test_gen/parser_tests/about_resource_field_present.ABOUT +++ b/tests/testdata/test_gen/parser_tests/about_resource_field_present.ABOUT @@ -2,9 +2,9 @@ name: Apache HTTP Server version: 2.4.3 date: 2012-08-21 license_spdx: Apache-2.0 -license_text_file: httpd.LICENSE +license_file: httpd.LICENSE copyright: Copyright 2012 The Apache Software Foundation. notice_file: httpd.NOTICE about_resource: about_resource.c -dje_license: apache-2.0 -dje_license_name: Apache License 2.0 \ No newline at end of file +license_expression: apache-2.0 +license_name: Apache License 2.0 \ No newline at end of file diff --git a/tests/testdata/test_model/android/expected_NOTICE b/tests/testdata/test_model/android/expected_NOTICE new file mode 100644 index 00000000..45b414e5 --- /dev/null +++ b/tests/testdata/test_model/android/expected_NOTICE @@ -0,0 +1,3 @@ +Copyright (c) xyz + +This component is released to the public domain by the author. \ No newline at end of file diff --git a/tests/testdata/test_model/android/multi_license.c.ABOUT b/tests/testdata/test_model/android/multi_license.c.ABOUT new file mode 100644 index 00000000..fb852db2 --- /dev/null +++ b/tests/testdata/test_model/android/multi_license.c.ABOUT @@ -0,0 +1,17 @@ +# Generated with AboutCode Toolkit Version 4.0.1 + +about_resource: multi_license.c +name: multi_license.c +license_expression: bsd-new OR bsd-simplified +copyright: | + Copyright (c) xyz + Copyright (c) Blah blah BlAh +licenses: + - key: bsd-new + name: BSD-3-Clause + file: bsd-new.LICENSE + url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new + - key: bsd-simplified + name: BSD-2-Clause + file: bsd-simplified.LICENSE + url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-simplified diff --git a/thirdparty/public-domain.LICENSE b/tests/testdata/test_model/android/public-domain.LICENSE similarity index 100% rename from thirdparty/public-domain.LICENSE rename to tests/testdata/test_model/android/public-domain.LICENSE diff --git a/tests/testdata/test_model/android/single_license.c.ABOUT b/tests/testdata/test_model/android/single_license.c.ABOUT new file mode 100644 index 00000000..1fba3e0b --- /dev/null +++ b/tests/testdata/test_model/android/single_license.c.ABOUT @@ -0,0 +1,11 @@ +# Generated with AboutCode Toolkit Version 4.0.1 + +about_resource: single_license.c +name: single_license.c +license_expression: public-domain +copyright: Copyright (c) xyz +licenses: + - key: public-domain + name: Public Domain + file: public-domain.LICENSE + url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain diff --git a/tests/testdata/test_model/dumps/about.ABOUT b/tests/testdata/test_model/dumps/about.ABOUT index 48e5cdc5..fd04182f 100644 --- a/tests/testdata/test_model/dumps/about.ABOUT +++ b/tests/testdata/test_model/dumps/about.ABOUT @@ -18,6 +18,7 @@ description: | license_expression: apache-2.0 license_key: apache-2.0 +license_name: Apache 2.0 license_file: apache-2.0.LICENSE copyright: Copyright (c) 2013-2014 nexB Inc. diff --git a/tests/testdata/test_model/inventory/complex/about/Jinja2.ABOUT b/tests/testdata/test_model/inventory/complex/about/Jinja2.ABOUT index 1fff666d..a6bd5dc4 100644 --- a/tests/testdata/test_model/inventory/complex/about/Jinja2.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/Jinja2.ABOUT @@ -5,8 +5,8 @@ download_url: https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.7.3.tar. name: Jinja2 homepage_url: http://jinja.pocoo.org/ -dje_license: bsd-new -license_text_file: Jinja2.LICENSE +license_expression: bsd-new +license_file: Jinja2.LICENSE vcs_tool: git vcs_repository: https://github.com/mitsuhiko/jinja2.git diff --git a/tests/testdata/test_model/inventory/complex/about/MarkupSafe.ABOUT b/tests/testdata/test_model/inventory/complex/about/MarkupSafe.ABOUT index dd631ddf..caf52719 100644 --- a/tests/testdata/test_model/inventory/complex/about/MarkupSafe.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/MarkupSafe.ABOUT @@ -5,11 +5,11 @@ download_url: https://pypi.python.org/packages/source/m/MarkupSafe/MarkupSafe-0. name: MarkupSafe homepage_url: https://github.com/mitsuhiko/markupsafe -dje_license: bsd-new +license_expression: bsd-new vcs_tool: git vcs_repository: https://github.com/mitsuhiko/jinja2.git -license_text_file: MarkupSafe.LICENSE +license_file: MarkupSafe.LICENSE copyright: Copyright (c) 2010 by Armin Ronacher and contributors. owner: Armin Ronacher diff --git a/tests/testdata/test_model/inventory/complex/about/certifi.ABOUT b/tests/testdata/test_model/inventory/complex/about/certifi.ABOUT index 37c0519b..536080e8 100644 --- a/tests/testdata/test_model/inventory/complex/about/certifi.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/certifi.ABOUT @@ -7,4 +7,4 @@ contact: me@kennethreitz.com homepage_url: http://python-requests.org name: certifi -dje_license: mpl-2.0 \ No newline at end of file +license_expression: mpl-2.0 \ No newline at end of file diff --git a/tests/testdata/test_model/inventory/complex/about/click.ABOUT b/tests/testdata/test_model/inventory/complex/about/click.ABOUT index c38195c3..8cd7fa2c 100644 --- a/tests/testdata/test_model/inventory/complex/about/click.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/click.ABOUT @@ -10,8 +10,8 @@ vcs_repository: https://github.com/mitsuhiko/click.git description: | A simple wrapper around optparse for powerful command line utilities. -dje_license: bsd-new -license_text_file: click.LICENSE +license_expression: bsd-new +license_file: click.LICENSE notes: | Click uses parts of optparse written by Gregory P. Ward and maintained by the Python software foundation. This is limited to code in the parser.py diff --git a/tests/testdata/test_model/inventory/complex/about/colorama.ABOUT b/tests/testdata/test_model/inventory/complex/about/colorama.ABOUT index 3a9a78e8..32e25d08 100644 --- a/tests/testdata/test_model/inventory/complex/about/colorama.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/colorama.ABOUT @@ -5,7 +5,7 @@ description: Cross-platform colored terminal text. homepage_url: https://pypi.python.org/pypi/colorama owner: Jonathan Hartley contact: tartley@tartley.com -dje_license: bds-new +license_expression: bds-new keywords: color colour terminal text ansi windows crossplatform xplatform -license_text_file: colorama.LICENSE +license_file: colorama.LICENSE download_url: https://pypi.python.org/packages/source/c/colorama/colorama-0.3.1.tar.gz#md5=95ce8bf32f5c25adea14b809db3509cb \ No newline at end of file diff --git a/tests/testdata/test_model/inventory/complex/about/pip.ABOUT b/tests/testdata/test_model/inventory/complex/about/pip.ABOUT index f8b6fa21..acaa073d 100644 --- a/tests/testdata/test_model/inventory/complex/about/pip.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/pip.ABOUT @@ -8,8 +8,8 @@ contact: python-virtualenv@groups.google.com homepage_url: http://www.pip-installer.org author_file: pip.AUTHORS -dje_license: mit, lgpl-2.1 -license_text_file: pip.LICENSE +license_expression: mit AND lgpl-2.1 +license_file: pip.LICENSE vcs_tool: git vcs_repository: https://github.com/pypa/pip.git \ No newline at end of file diff --git a/tests/testdata/test_model/inventory/complex/about/py.ABOUT b/tests/testdata/test_model/inventory/complex/about/py.ABOUT index cdc78953..968c7c56 100644 --- a/tests/testdata/test_model/inventory/complex/about/py.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/py.ABOUT @@ -7,8 +7,8 @@ description: library with cross-python path, ini-parsing, io, code, log faciliti homepage_url: http://pylib.readthedocs.org/ owner: holger krekel, Ronny Pfannschmidt, Benjamin Peterson and others contact: pytest-dev@python.org -dje_license: mit -license_text_file: py.LICENSE +license_expression: mit +license_file: py.LICENSE copyright: Holger Krekel and others, 2004-2014 diff --git a/tests/testdata/test_model/inventory/complex/about/pytest.ABOUT b/tests/testdata/test_model/inventory/complex/about/pytest.ABOUT index bc05ec02..5bbecee4 100644 --- a/tests/testdata/test_model/inventory/complex/about/pytest.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/pytest.ABOUT @@ -7,6 +7,6 @@ description: pytest - simple powerful testing with Python homepage_url: http://pytest.org owner: Holger Krekel, Benjamin Peterson, Ronny Pfannschmidt, Floris Bruynooghe and others contact: holger at merlinux.eu -dje_license: mit -license_text_file: pytest.LICENSE +license_expression: mit +license_file: pytest.LICENSE copyright: Copyright Holger Krekel and others, 2004-2014 diff --git a/tests/testdata/test_model/inventory/complex/about/schematics.ABOUT b/tests/testdata/test_model/inventory/complex/about/schematics.ABOUT index 8fdaf90a..c95630c2 100644 --- a/tests/testdata/test_model/inventory/complex/about/schematics.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/schematics.ABOUT @@ -8,5 +8,5 @@ homepage_url: https://github.com/schematics/schematics owner: J2 Labs LLC. copyright: Copyright (c) 2013, J2 Labs LLC. -dje_license: bsd-new -license_text_file: schematics.LICENSE \ No newline at end of file +license_expression: bsd-new +license_file: schematics.LICENSE \ No newline at end of file diff --git a/tests/testdata/test_model/inventory/complex/about/setuptools.ABOUT b/tests/testdata/test_model/inventory/complex/about/setuptools.ABOUT index 80231477..bb744412 100644 --- a/tests/testdata/test_model/inventory/complex/about/setuptools.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/setuptools.ABOUT @@ -7,5 +7,5 @@ download_url: https://pypi.python.org/packages/3.4/s/setuptools/setuptools-5.6-p homepage_url: https://pypi.python.org/pypi/setuptools owner: Python Packaging Authority -dje_license: psf -license_text_file: PSF.LICENSE \ No newline at end of file +license_expression: psf +license_file: PSF.LICENSE \ No newline at end of file diff --git a/tests/testdata/test_model/inventory/complex/about/unicodecsv.ABOUT b/tests/testdata/test_model/inventory/complex/about/unicodecsv.ABOUT index 932c01e3..6107fbca 100644 --- a/tests/testdata/test_model/inventory/complex/about/unicodecsv.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/unicodecsv.ABOUT @@ -6,8 +6,8 @@ name: unicodecsv homepage_url: https://github.com/jdunck/python-unicodecsv owner: Jeremy Dunck -dje_license: bsd-new -license_text_file: unicodecsv.LICENSE +license_expression: bsd-new +license_file: unicodecsv.LICENSE vcs_tool: git vcs_repository: https://github.com/jdunck/python-unicodecsv.git diff --git a/tests/testdata/test_model/inventory/complex/about/virtualenv.ABOUT b/tests/testdata/test_model/inventory/complex/about/virtualenv.ABOUT index 79170066..2f3d254e 100644 --- a/tests/testdata/test_model/inventory/complex/about/virtualenv.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/virtualenv.ABOUT @@ -10,8 +10,8 @@ homepage_url: http://virtualenv.org/ owner: The virtualenv developers license_url: https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt -license_text_file: virtualenv.LICENSE -dje_license: mit +license_file: virtualenv.LICENSE +license_expression: mit copyright: | Copyright (c) 2007 Ian Bicking and Contributors Copyright (c) 2009 Ian Bicking, The Open Planning Project diff --git a/tests/testdata/test_model/inventory/complex/about/virtualenv.py.ABOUT b/tests/testdata/test_model/inventory/complex/about/virtualenv.py.ABOUT index b3192200..603ea0b6 100644 --- a/tests/testdata/test_model/inventory/complex/about/virtualenv.py.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/virtualenv.py.ABOUT @@ -10,8 +10,8 @@ homepage_url: http://virtualenv.org/ owner: The virtualenv developers license_url: https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt -license_text_file: virtualenv.LICENSE -dje_license: mit +license_file: virtualenv.LICENSE +license_expression: mit copyright: | Copyright (c) 2007 Ian Bicking and Contributors Copyright (c) 2009 Ian Bicking, The Open Planning Project diff --git a/tests/testdata/test_model/inventory/complex/about/wheel.ABOUT b/tests/testdata/test_model/inventory/complex/about/wheel.ABOUT index 338c37da..0ac2fec2 100644 --- a/tests/testdata/test_model/inventory/complex/about/wheel.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/wheel.ABOUT @@ -10,5 +10,5 @@ vcs_repository: https://bitbucket.org/pypa/wheel copyright: | copyright (c) 2012-2014 Daniel Holth and contributors. -dje_license: mit -license_text_file: wheel.LICENSE +license_expression: mit +license_file: wheel.LICENSE diff --git a/tests/testdata/test_model/inventory/complex/about/wincertstore.ABOUT b/tests/testdata/test_model/inventory/complex/about/wincertstore.ABOUT index 634c43ce..bd062f06 100644 --- a/tests/testdata/test_model/inventory/complex/about/wincertstore.ABOUT +++ b/tests/testdata/test_model/inventory/complex/about/wincertstore.ABOUT @@ -4,8 +4,8 @@ download_url: https://pypi.python.org/packages/source/w/wincertstore/wincertstor name: wincertstore -dje_license: psf -license_text_file: wincertstore.LICENSE +license_expression: psf +license_file: wincertstore.LICENSE contact: christian@python.org owner: Christian Heimes diff --git a/tests/testdata/test_model/inventory/complex/expected.csv b/tests/testdata/test_model/inventory/complex/expected.csv index 4c103098..46f312ee 100644 --- a/tests/testdata/test_model/inventory/complex/expected.csv +++ b/tests/testdata/test_model/inventory/complex/expected.csv @@ -1,9 +1,9 @@ -about_resource,name,version,download_url,description,homepage_url,notes,license_file,license_url,copyright,notice_file,owner,contact,author,author_file,vcs_tool,vcs_repository,dje_license,keywords,license,license_text_file +about_resource,name,version,download_url,description,homepage_url,notes,license_file,license_url,copyright,notice_file,owner,contact,author,author_file,vcs_tool,vcs_repository,license_expression,keywords,license,license_file /about/pytest-2.6.1-py2.py3-none-any.whl,pytest,2.6.1,https://pypi.python.org/packages/source/p/pytest/pytest-2.6.1.tar.gz#md5=bb353f6cf6d9ff83ff7f2dfbeaca47a3,pytest - simple powerful testing with Python,http://pytest.org,,,,"Copyright Holger Krekel and others, 2004-2014",,"Holger Krekel, Benjamin Peterson, Ronny Pfannschmidt, Floris Bruynooghe and others",holger at merlinux.eu,,,,,mit,,,pytest.LICENSE /about/Jinja2-2.7.3-py2-none-any.whl,Jinja2,2.7.3,https://pypi.python.org/packages/source/J/Jinja2/Jinja2-2.7.3.tar.gz#md5=b9dffd2f3b43d673802fe857c8445b1a,,http://jinja.pocoo.org/,,,,Copyright (c) 2009 by the Jinja Team,,Armin Ronacher,,,,git,https://github.com/mitsuhiko/jinja2.git,bsd-new,,,Jinja2.LICENSE /about/,AboutCode,0.11.0,,"AboutCode is a tool to process ABOUT files. -An ABOUT file is a file.",http://dejacode.org,,apache-2.0.LICENSE,,Copyright (c) 2013-2014 nexB Inc.,NOTICE,nexB Inc.,,"Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez",,git,https://github.com/dejacode/about-code-tool.git,,,apache-2.0, +An ABOUT file is a file.",http://dejacode.org,,apache-2.0.LICENSE,,Copyright (c) 2013-2014 nexB Inc.,NOTICE,nexB Inc.,,"Jillian Daguil, Chin Yeung Li, Philippe Ombredanne, Thomas Druez",,git,https://github.com/dejacode/about-code-tool.git,,,apache-2.0,apache-2.0.LICENSE /about/virtualenv.py,virtualenv,1.11.6,https://raw.githubusercontent.com/pypa/virtualenv/1.11.6/virtualenv.py,,http://virtualenv.org/,,,https://raw.github.com/pypa/virtualenv/develop/LICENSE.txt,"Copyright (c) 2007 Ian Bicking and Contributors Copyright (c) 2009 Ian Bicking, The Open Planning Project Copyright (c) 2011-2014 The virtualenv developers",,The virtualenv developers,,,,git,https://github.com/pypa/virtualenv.git,mit,,,virtualenv.LICENSE @@ -23,5 +23,5 @@ module and is under the same license as clikc itself.",,,,,Armin Ronacher,armin. /about/wheel-0.24.0-py2.py3-none-any.whl,wheel,0.24.0,https://pypi.python.org/packages/py2.py3/w/wheel/wheel-0.24.0-py2.py3-none-any.whl#md5=4c24453cda2177fd42c5d62d6434679a,,https://bitbucket.org/pypa/wheel,,,,"copyright (c) 2012-2014 Daniel Holth and contributors.",,,,,,hg,https://bitbucket.org/pypa/wheel,mit,,,wheel.LICENSE /about/unicodecsv-0.9.4-py2-none-any.whl,unicodecsv,0.9.4,https://pypi.python.org/packages/source/u/unicodecsv/unicodecsv-0.9.4.tar.gz#md5=344fa55f299ba198cb73db48546002fd,,https://github.com/jdunck/python-unicodecsv,,,,,,Jeremy Dunck,,,,git,https://github.com/jdunck/python-unicodecsv.git,bsd-new,,,unicodecsv.LICENSE -/about/pip-1.5.6-py2.py3-none-any.whl,pip,1.5.6,https://pypi.python.org/packages/source/p/pip/pip-1.5.6.tar.gz#md5=01026f87978932060cc86c1dc527903e,,http://www.pip-installer.org,,,,,,The pip developers,python-virtualenv@groups.google.com,,pip.AUTHORS,git,https://github.com/pypa/pip.git,"mit, lgpl-2.1",,,pip.LICENSE +/about/pip-1.5.6-py2.py3-none-any.whl,pip,1.5.6,https://pypi.python.org/packages/source/p/pip/pip-1.5.6.tar.gz#md5=01026f87978932060cc86c1dc527903e,,http://www.pip-installer.org,,,,,,The pip developers,python-virtualenv@groups.google.com,,pip.AUTHORS,git,https://github.com/pypa/pip.git,mit AND lgpl-2.1,,,pip.LICENSE /about/py-1.4.23-py2-none-any.whl,py,1.4.23,https://pypi.python.org/packages/source/p/py/py-1.4.23.tar.gz#md5=b40aea711eeb8adba0c44f0b750a3205,"library with cross-python path, ini-parsing, io, code, log facilities",http://pylib.readthedocs.org/,,,,"Holger Krekel and others, 2004-2014",,"holger krekel, Ronny Pfannschmidt, Benjamin Peterson and others",pytest-dev@python.org,,,,,mit,,,py.LICENSE diff --git a/tests/testdata/test_model/parse/boolean_chara_data.about b/tests/testdata/test_model/parse/boolean_chara_data.about new file mode 100644 index 00000000..f1b0bac5 --- /dev/null +++ b/tests/testdata/test_model/parse/boolean_chara_data.about @@ -0,0 +1,3 @@ +about_resource: . +name: data +attribute: 11 diff --git a/tests/testdata/test_model/parse/boolean_more_than_2_chara_data.about b/tests/testdata/test_model/parse/boolean_more_than_2_chara_data.about new file mode 100644 index 00000000..4ccf4619 --- /dev/null +++ b/tests/testdata/test_model/parse/boolean_more_than_2_chara_data.about @@ -0,0 +1,3 @@ +about_resource: . +name: test +attribute: abc diff --git a/tests/testdata/test_model/parse/boolean_numeric_data.about b/tests/testdata/test_model/parse/boolean_numeric_data.about new file mode 100644 index 00000000..627f95f0 --- /dev/null +++ b/tests/testdata/test_model/parse/boolean_numeric_data.about @@ -0,0 +1,7 @@ +about_resource: . +name: boolean_data +attribute: 3 +modified: true +internal_use_only: no +redistribute: yes +track_changes: diff --git a/tests/testdata/test_model/parse/empty_required.ABOUT b/tests/testdata/test_model/parse/empty_required.ABOUT index bd38df56..a818778e 100644 --- a/tests/testdata/test_model/parse/empty_required.ABOUT +++ b/tests/testdata/test_model/parse/empty_required.ABOUT @@ -1,2 +1 @@ -name: -about_resource: +name: diff --git a/tests/testdata/test_model/parse/with_ignored_resources.ABOUT b/tests/testdata/test_model/parse/with_ignored_resources.ABOUT new file mode 100644 index 00000000..c71bb338 --- /dev/null +++ b/tests/testdata/test_model/parse/with_ignored_resources.ABOUT @@ -0,0 +1,5 @@ +name: elasticsearch-sidecar +about_resource: elasticsearch-sidecar +ignored_resources: + - elasticsearch-sidecar/plugins/ + - elasticsearch-sidecar/logs/ diff --git a/tests/testdata/test_model/redistribution/test/subdir/test.ABOUT b/tests/testdata/test_model/redistribution/test/subdir/test.ABOUT new file mode 100644 index 00000000..44bbc404 --- /dev/null +++ b/tests/testdata/test_model/redistribution/test/subdir/test.ABOUT @@ -0,0 +1,3 @@ +about_resource: . +name: test +redistribute: x \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/virtualenv-1.11.6-py2.py3-none-any.whl b/tests/testdata/test_model/redistribution/test/subdir/test.c similarity index 100% rename from tests/testdata/test_gen/inventory/complex/about/virtualenv-1.11.6-py2.py3-none-any.whl rename to tests/testdata/test_model/redistribution/test/subdir/test.c diff --git a/tests/testdata/test_model/redistribution/this.ABOUT b/tests/testdata/test_model/redistribution/this.ABOUT new file mode 100644 index 00000000..b9f466c4 --- /dev/null +++ b/tests/testdata/test_model/redistribution/this.ABOUT @@ -0,0 +1,4 @@ +about_resource: this.c +version: 0.11.0 +name: this.c +redistribute: x \ No newline at end of file diff --git a/tests/testdata/test_gen/inventory/complex/about/virtualenv.py b/tests/testdata/test_model/redistribution/this.c similarity index 100% rename from tests/testdata/test_gen/inventory/complex/about/virtualenv.py rename to tests/testdata/test_model/redistribution/this.c diff --git a/tests/testdata/test_model/redistribution/this2.ABOUT b/tests/testdata/test_model/redistribution/this2.ABOUT new file mode 100644 index 00000000..1ee8c99e --- /dev/null +++ b/tests/testdata/test_model/redistribution/this2.ABOUT @@ -0,0 +1,4 @@ +about_resource: this2.c +name: this2.c +version: 0.11.0 +redistribute: N \ No newline at end of file diff --git a/tests/testdata/test_model/redistribution/this2.c b/tests/testdata/test_model/redistribution/this2.c new file mode 100644 index 00000000..e69de29b diff --git a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/csv_serialize.py.ABOUT b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/csv_serialize.py.ABOUT index 31b1c16c..7ab52de0 100644 --- a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/csv_serialize.py.ABOUT +++ b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/csv_serialize.py.ABOUT @@ -3,6 +3,6 @@ name: csv_serialize version: 2013-01-16 homepage_url: http://djangosnippets.org/snippets/2240/ license_url: http://djangosnippets.org/about/tos/ -license_text_file: ../../thirdparty/django_snippets.LICENSE +license_file: ../../thirdparty/django_snippets.LICENSE about_resource: ../../thirdparty/csv_serialize.py \ No newline at end of file diff --git a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/django_snippets_2413.ABOUT b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/django_snippets_2413.ABOUT index a59ebafc..915ca350 100644 --- a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/django_snippets_2413.ABOUT +++ b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/django_snippets_2413.ABOUT @@ -6,6 +6,6 @@ name: Yet another query string template tag homepage_url: http://djangosnippets.org/snippets/2413/ license_url: http://djangosnippets.org/about/tos/ -license_text_file: ../../thirdparty/django_snippets.LICENSE +license_file: ../../thirdparty/django_snippets.LICENSE notes: This file was modified to include the line "register = Library()" without which the template tag is not registered. diff --git a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/elasticsearch.ABOUT b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/elasticsearch.ABOUT index 5051e41b..55c909b4 100644 --- a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/elasticsearch.ABOUT +++ b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/elasticsearch.ABOUT @@ -11,9 +11,9 @@ homepage_url: http://www.elasticsearch.org/ scm_tool: git scm_repository: https://github.com/elasticsearch/elasticsearch.git -dje_license: apache-2.0 +license_expression: apache-2.0 notice_file: ../../thirdparty/elasticsearch.NOTICE -license_text_file: ../../thirdparty/elasticsearch.LICENSE +license_file: ../../thirdparty/elasticsearch.LICENSE copyright: Copyright 2009-2011 ElasticSearch and Shay Banon notes: This a prebuilt version working on all OSes. diff --git a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/ez_setup.py.ABOUT b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/ez_setup.py.ABOUT index e84dd279..363f4a7b 100644 --- a/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/ez_setup.py.ABOUT +++ b/tests/testdata/test_model/rel/allAboutInOneDir/about_ref/ez_setup.py.ABOUT @@ -6,7 +6,7 @@ name: setuptools boostrap homepage_url: http://pypi.python.org/pypi/setuptools author: Phillip J. Eby -dje_license: zpl-2.1 +license_expression: zpl-2.1 notes: this is not used by default but embedded in virtualenv about_resource: t1/t2/ez_setup.py \ No newline at end of file diff --git a/tests/testdata/test_model/rel/thirdparty/django_snippets.LICENSE b/tests/testdata/test_model/rel/thirdparty/django_snippets.LICENSE new file mode 100644 index 00000000..e69de29b diff --git a/tests/testdata/test_model/rel/thirdparty/elasticsearch.LICENSE b/tests/testdata/test_model/rel/thirdparty/elasticsearch.LICENSE new file mode 100644 index 00000000..e69de29b diff --git a/tests/testdata/test_model/single_file/django_snippets_2413.ABOUT b/tests/testdata/test_model/single_file/django_snippets_2413.ABOUT index a8448919..20f49c9c 100644 --- a/tests/testdata/test_model/single_file/django_snippets_2413.ABOUT +++ b/tests/testdata/test_model/single_file/django_snippets_2413.ABOUT @@ -6,6 +6,6 @@ name: Yet another query string template tag homepage_url: http://djangosnippets.org/snippets/2413/ license_url: http://djangosnippets.org/about/tos/ -license_text_file: django_snippets.LICENSE +license_file: django_snippets.LICENSE notes: This file was modified to include the line "register = Library()" without which the template tag is not registered. diff --git a/tests/testdata/test_model/special_char/about.ABOUT b/tests/testdata/test_model/special_char/about.ABOUT new file mode 100644 index 00000000..30e18b28 --- /dev/null +++ b/tests/testdata/test_model/special_char/about.ABOUT @@ -0,0 +1,6 @@ +about_resource: . + +name: AboutCode +version: 0.11.0 + +license_key: mit, gpl-2.0 \ No newline at end of file diff --git a/tests/testdata/test_transform/configuration b/tests/testdata/test_transform/configuration index e650d4f0..9af1cdfd 100644 --- a/tests/testdata/test_transform/configuration +++ b/tests/testdata/test_transform/configuration @@ -1,3 +1,13 @@ -column_renamings: - 'Directory/Filename' : about_resource - Component: name \ No newline at end of file +field_renamings: + about_resource : 'Directory/Filename' + name : Component +field_filters: + - about_resource + - name + - version + - temp +required_fields: + - name + - version +exclude_fields: + - temp diff --git a/tests/testdata/test_transform/configuration2 b/tests/testdata/test_transform/configuration2 new file mode 100644 index 00000000..fe08089c --- /dev/null +++ b/tests/testdata/test_transform/configuration2 @@ -0,0 +1,4 @@ +field_renamings: + about_resource : 'Directory/Filename' + name : Component + version : 'Confirmed Version' \ No newline at end of file diff --git a/tests/testdata/test_transform/configuration3 b/tests/testdata/test_transform/configuration3 new file mode 100644 index 00000000..ff51de1d --- /dev/null +++ b/tests/testdata/test_transform/configuration3 @@ -0,0 +1,15 @@ +field_renamings: + about_resource : 'path' + score_renamed : score + size_renamed : size +required_fields: + - about_resource + - name +exclude_fields: + - sha1 + - sha256 + - md5 + - type + - start_line + - matched_length + - scan_errors diff --git a/tests/testdata/test_transform/configuration_new_cols b/tests/testdata/test_transform/configuration_new_cols new file mode 100644 index 00000000..80cd5fab --- /dev/null +++ b/tests/testdata/test_transform/configuration_new_cols @@ -0,0 +1,4 @@ +field_renamings: + about_resource: 'Directory/Filename' + name: Component + path: 'Directory/Filename' \ No newline at end of file diff --git a/tests/testdata/test_transform/input.csv b/tests/testdata/test_transform/input.csv index b98cda98..801846ad 100644 --- a/tests/testdata/test_transform/input.csv +++ b/tests/testdata/test_transform/input.csv @@ -1,2 +1,2 @@ -Directory/Filename,Component -/tmp/test.c, test,c +Directory/Filename,Component,Confirmed Version,notes +/aboutcode-toolkit/,AboutCode-toolkit,123, diff --git a/tests/testdata/test_transform/input.json b/tests/testdata/test_transform/input.json new file mode 100644 index 00000000..e8015800 --- /dev/null +++ b/tests/testdata/test_transform/input.json @@ -0,0 +1,8 @@ +[ + { + "Directory/Filename": "/aboutcode-toolkit/", + "Component": "AboutCode-toolkit", + "Confirmed Version": "123", + "notes": "" + } +] \ No newline at end of file diff --git a/tests/testdata/test_transform/input.xlsx b/tests/testdata/test_transform/input.xlsx new file mode 100644 index 00000000..e9ec15e2 Binary files /dev/null and b/tests/testdata/test_transform/input.xlsx differ diff --git a/tests/testdata/test_transform/input_scancode.json b/tests/testdata/test_transform/input_scancode.json new file mode 100644 index 00000000..58ecf2d4 --- /dev/null +++ b/tests/testdata/test_transform/input_scancode.json @@ -0,0 +1,62 @@ +{ + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "3.1.1", + "options": { + "input": [ + "samples" + ], + "--copyright": true, + "--email": true, + "--info": true, + "--json-pp": "output.json", + "--license": true, + "--package": true, + "--url": true + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2020-03-10T072841.008622", + "end_timestamp": "2020-03-10T072953.529915", + "message": null, + "errors": [], + "extra_data": { + "files_count": 33 + } + } + ], + "files": [ + { + "path": "samples", + "type": "directory", + "name": "samples", + "base_name": "samples", + "extension": "", + "size": 0, + "date": null, + "sha1": null, + "md5": null, + "mime_type": null, + "file_type": null, + "programming_language": null, + "is_binary": false, + "is_text": false, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "licenses": [], + "license_expressions": [], + "copyrights": [], + "holders": [], + "authors": [], + "packages": [], + "emails": [], + "urls": [], + "files_count": 33, + "dirs_count": 10, + "size_count": 1161083, + "scan_errors": [] + } + ] +} diff --git a/tests/testdata/test_transform/simple.csv b/tests/testdata/test_transform/simple.csv new file mode 100644 index 00000000..3fcae6b2 --- /dev/null +++ b/tests/testdata/test_transform/simple.csv @@ -0,0 +1,3 @@ +about_resource,name,license_expression +/test.c,test.c,mit +/test2.c,test2.c,mit and apache-2.0 diff --git a/tests/testdata/test_transform/simple.xlsx b/tests/testdata/test_transform/simple.xlsx new file mode 100644 index 00000000..3701901f Binary files /dev/null and b/tests/testdata/test_transform/simple.xlsx differ diff --git a/tests/testdata/test_util/csv/test_ms_utf8.csv b/tests/testdata/test_util/csv/test_ms_utf8.csv new file mode 100644 index 00000000..7939d92a --- /dev/null +++ b/tests/testdata/test_util/csv/test_ms_utf8.csv @@ -0,0 +1,2 @@ +about_resource,name +/myFile,myName diff --git a/tests/testdata/test_util/csv/test_utf8.csv b/tests/testdata/test_util/csv/test_utf8.csv new file mode 100644 index 00000000..c20550cd --- /dev/null +++ b/tests/testdata/test_util/csv/test_utf8.csv @@ -0,0 +1,2 @@ +about_resource,name +/myFile,名 diff --git a/tests/testdata/test_util/json/aboutcode_manager_exported.json b/tests/testdata/test_util/json/aboutcode_manager_exported.json deleted file mode 100644 index 0a46cbf9..00000000 --- a/tests/testdata/test_util/json/aboutcode_manager_exported.json +++ /dev/null @@ -1,29 +0,0 @@ -{ -"aboutcode_manager_notice":"Exported from AboutCode Manager and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\\nAboutCode Manager should be considered or used as legal advice. Consult an Attorney\\nfor any legal advice.\\nAboutCode Manager is a free software analysis application from nexB Inc. and others.\\nVisit https://github.com/nexB/aboutcode-manager/ for support and download.\"", -"aboutcode_manager_version":"2.4.0", - -"components": -[{"license_expression":"apache-2.0", -"copyright":"Copyright (c) 2017 nexB Inc.", -"licenses":[{"key":"apache-2.0"}], -"copyrights":[{"statements":["Copyright (c) 2017 nexB Inc."]}], -"path":"ScanCode", -"review_status":"Analyzed", -"name":"ScanCode", -"version":"2.2.1", -"owner":"nexB Inc.", -"code_type":"Source", -"is_modified":false, -"is_deployed":false, -"feature":"", -"purpose":"", -"homepage_url":null, -"download_url":null, -"license_url":null, -"notice_url":null, -"programming_language":"Python", -"notes":"", -"fileId":8458 -}] - -} \ No newline at end of file diff --git a/tests/testdata/test_util/json/multi_entries.json b/tests/testdata/test_util/json/multi_entries.json new file mode 100644 index 00000000..80fdde29 --- /dev/null +++ b/tests/testdata/test_util/json/multi_entries.json @@ -0,0 +1,13 @@ +[ + { + "about_file_path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0" + }, + { + "about_file_path": "/load/that.ABOUT", + "about_resource": ".", + "name": "that" + } +] diff --git a/tests/testdata/test_util/json/not_a_list.json b/tests/testdata/test_util/json/not_a_list.json index 5596d01c..d23b8071 100644 --- a/tests/testdata/test_util/json/not_a_list.json +++ b/tests/testdata/test_util/json/not_a_list.json @@ -1,6 +1,6 @@ { - "about_file_path": "/load/this.ABOUT", - "about_resource": ".", - "name": "AboutCode", - "version": "0.11.0" -} \ No newline at end of file + "about_file_path": "/load/this.ABOUT", + "about_resource": ".", + "name": "AboutCode", + "version": "0.11.0" +} diff --git a/tests/testdata/test_util/json/scancode_info.json b/tests/testdata/test_util/json/scancode_info.json index 288fcf5e..7cfd1fda 100644 --- a/tests/testdata/test_util/json/scancode_info.json +++ b/tests/testdata/test_util/json/scancode_info.json @@ -1,31 +1,56 @@ { - "scancode_notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", - "scancode_version": "2.9.0b1.post32.fea65d3", - "scancode_options": { - "input": "C:\\Users\\CYL\\Downloads\\io\\swagger\\annotations\\Api.java", - "--info": true, - "--json-pp": "C:\\Users\\CYL\\Desktop\\test\\scancode-info.json" - }, - "files_count": 1, + "headers": [ + { + "tool_name": "scancode-toolkit", + "tool_version": "v32.0.6-2-g200fb10eb2", + "options": { + "input": [ + "C:\\lic\\lic.txt" + ], + "--info": true, + "--json-pp": "-" + }, + "notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.", + "start_timestamp": "2023-08-01T053742.737068", + "end_timestamp": "2023-08-01T053744.254262", + "output_format_version": "3.0.0", + "duration": 1.5171935558319092, + "message": null, + "errors": [], + "warnings": [], + "extra_data": { + "system_environment": { + "operating_system": "win", + "cpu_architecture": "64", + "platform": "Windows-10-10.0.22621-SP0", + "platform_version": "10.0.22621", + "python_version": "3.9.0 (tags/v3.9.0:9cf6752, Oct 5 2020, 15:34:40) [MSC v.1927 64 bit (AMD64)]" + }, + "spdx_license_list_version": "3.21", + "files_count": 1 + } + } + ], "files": [ { - "path": "Api.java", + "path": "lic.txt", "type": "file", - "name": "Api.java", - "base_name": "Api", - "extension": ".java", - "size": 5074, - "date": "2017-07-15", - "sha1": "c3a48ec7e684a35417241dd59507ec61702c508c", - "md5": "326fb262bbb9c2ce32179f0450e24601", + "name": "lic.txt", + "base_name": "lic", + "extension": ".txt", + "size": 1463, + "date": "2023-07-26", + "sha1": "bb3f381f9ec25416c0c3b4628f7f6b923ced040f", + "md5": "63f9ec8c32874a5d987d78b9a730a6b8", + "sha256": "d71777b3dc333f540a871bf2ef6380e646a10f2ac1f077ce4f34326e16fb6995", "mime_type": "text/plain", - "file_type": "ASCII text", - "programming_language": "Java", + "file_type": "ASCII text, with very long lines", + "programming_language": null, "is_binary": false, "is_text": true, "is_archive": false, "is_media": false, - "is_source": true, + "is_source": false, "is_script": false, "files_count": 0, "dirs_count": 0, diff --git a/tests/testdata/test_util/licenses/mit.LICENSE b/tests/testdata/test_util/licenses/mit.LICENSE new file mode 100644 index 00000000..108d51b5 --- /dev/null +++ b/tests/testdata/test_util/licenses/mit.LICENSE @@ -0,0 +1,5 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.This component is released to the public domain by the author. \ No newline at end of file diff --git a/tests/testdata/test_util/licenses/mit2.LICENSE b/tests/testdata/test_util/licenses/mit2.LICENSE new file mode 100644 index 00000000..108d51b5 --- /dev/null +++ b/tests/testdata/test_util/licenses/mit2.LICENSE @@ -0,0 +1,5 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.This component is released to the public domain by the author. \ No newline at end of file diff --git a/tests/testdata/test_util/licenses/public-domain.LICENSE b/tests/testdata/test_util/licenses/public-domain.LICENSE new file mode 100644 index 00000000..e5c890da --- /dev/null +++ b/tests/testdata/test_util/licenses/public-domain.LICENSE @@ -0,0 +1 @@ +This component is released to the public domain by the author. \ No newline at end of file diff --git a/tests/testing_utils.py b/tests/testing_utils.py index fb15147a..4b0764ff 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -2,7 +2,7 @@ # -*- coding: utf8 -*- # ============================================================================ -# Copyright (c) 2017 nexB Inc. http://www.nexb.com/ - All rights reserved. +# Copyright (c) nexB Inc. http://www.nexb.com/ - All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,10 +14,6 @@ # limitations under the License. # ============================================================================ -from __future__ import absolute_import -from __future__ import print_function -from __future__ import unicode_literals - import logging import ntpath import os @@ -28,20 +24,17 @@ import tempfile import zipfile - from attributecode.util import add_unc from attributecode.util import to_posix - logger = logging.getLogger(__name__) handler = logging.StreamHandler() handler.setLevel(logging.DEBUG) logger.addHandler(handler) +TESTDATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "testdata") -TESTDATA_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'testdata') - -on_windows = 'win32' in sys.platform +on_windows = "win32" in sys.platform on_posix = not on_windows @@ -57,6 +50,7 @@ def get_test_loc(path, must_exists=True): assert os.path.exists(path) return path + def create_dir(location): """ Create directory or directory tree at location, ensuring it is readable @@ -64,11 +58,10 @@ def create_dir(location): """ if not os.path.exists(location): os.makedirs(location) - os.chmod(location, stat.S_IRWXU | stat.S_IRWXG - | stat.S_IROTH | stat.S_IXOTH) + os.chmod(location, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH) -def build_temp_dir(prefix='test-attributecode-'): +def build_temp_dir(prefix="test-attributecode-"): """ Create and return a new unique empty directory created in base_dir. """ @@ -77,7 +70,7 @@ def build_temp_dir(prefix='test-attributecode-'): return location -def get_temp_file(file_name='test-attributecode-tempfile'): +def get_temp_file(file_name="test-attributecode-tempfile"): """ Return a unique new temporary file location to a non-existing temporary file that can safely be created without a risk of name @@ -109,7 +102,7 @@ def extract_zip(location, target_dir): Extract a zip archive file at location in the target_dir directory. """ if not os.path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: for info in zipf.infolist(): @@ -124,7 +117,7 @@ def extract_zip(location, target_dir): if not os.path.exists(target): os.makedirs(target) if not os.path.exists(target): - with open(target, 'wb') as f: + with open(target, "wb") as f: f.write(content) @@ -150,30 +143,27 @@ def run_about_command_test(options, expected_rc=0): On success, return stdout and stderr. """ root_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - about_cmd = os.path.join(root_dir, 'about') + about_cmd = os.path.join(root_dir, "about") args = [about_cmd] + options about = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True if on_windows else False) + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True if on_windows else False + ) stdout, stderr = about.communicate() rc = about.poll() if rc != expected_rc: - opts = ' '.join(args) + opts = " ".join(args) error = ( - 'Failure to run command: %(opts)s\n' - 'stdout:\n' - '{stdout}\n' - '\n' - 'stderr:\n' - '{stderr}\n' + "Failure to run command: %(opts)s\nstdout:\n{stdout}\n\nstderr:\n{stderr}\n" ).format(**locals()) assert rc == expected_rc, error return stdout, stderr -def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): +def run_about_command_test_click( + options, + expected_rc=0, + monkeypatch=None, +): """ Run an "about" command as a Click-controlled subprocess with the `options` list of options. Return a click.testing.Result object. @@ -183,9 +173,17 @@ def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): import click from click.testing import CliRunner from attributecode import cmd + if monkeypatch: - monkeypatch.setattr(click._termui_impl, 'isatty', lambda _: True) - monkeypatch.setattr(click , 'get_terminal_size', lambda : (80, 43,)) + monkeypatch.setattr(click._termui_impl, "isatty", lambda _: True) + monkeypatch.setattr( + click, + "get_terminal_size", + lambda: ( + 80, + 43, + ), + ) runner = CliRunner() result = runner.invoke(cmd.about, options, catch_exceptions=False) @@ -193,20 +191,20 @@ def run_about_command_test_click(options, expected_rc=0, monkeypatch=None,): output = result.output if result.exit_code != expected_rc: opts = get_opts(options) - error = ''' + error = """ Failure to run: about %(opts)s output: %(output)s -''' % locals() +""" % locals() assert result.exit_code == expected_rc, error return result def get_opts(options): try: - return ' '.join(options) + return " ".join(options) except: try: - return b' '.join(options) + return b" ".join(options) except: - return b' '.join(map(repr, options)) + return b" ".join(map(repr, options)) diff --git a/thirdparty/Jinja2-2.10-py2.py3-none-any.whl b/thirdparty/Jinja2-2.10-py2.py3-none-any.whl deleted file mode 100644 index 7bc4e35f..00000000 Binary files a/thirdparty/Jinja2-2.10-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/Jinja2-2.10-py2.py3-none-any.whl.ABOUT b/thirdparty/Jinja2-2.10-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 704297fb..00000000 --- a/thirdparty/Jinja2-2.10-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,20 +0,0 @@ -about_resource: Jinja2-2.10-py2.py3-none-any.whl -name: Jinja2 -version: '2.1' -checksum_md5: cb679acd14423aef56dfff61d6a988f8 -checksum_sha1: f14bb3e5021b2b0605dcd4865ac539cde04fe856 -contact: armin.ronacher@active-4.com -copyright: Copyright (c) 2009 by the Jinja Team -description: | - Jinja2 is a full featured template engine for Python. It has full unicode support, - an optional integrated sandboxed execution environment, widely used and BSD licensed. -download_url: https://pypi.python.org/packages/7f/ff/ae64bacdfc95f27a016a7bed8e8686763ba4d277a78ca76f32659220a731/Jinja2-2.10-py2.py3-none-any.whl#md5=cb679acd14423aef56dfff61d6a988f8 -homepage_url: http://jinja.pocoo.org/ -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Armin Ronacher -owner_url: http://lucumr.pocoo.org/about/ diff --git a/thirdparty/Jinja2.LICENSE b/thirdparty/Jinja2.LICENSE deleted file mode 100644 index 5a0acde8..00000000 --- a/thirdparty/Jinja2.LICENSE +++ /dev/null @@ -1,31 +0,0 @@ -Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/MarkupSafe-1.0-py2-none-any.whl b/thirdparty/MarkupSafe-1.0-py2-none-any.whl deleted file mode 100644 index e003df2b..00000000 Binary files a/thirdparty/MarkupSafe-1.0-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/MarkupSafe-1.0-py2-none-any.whl.ABOUT b/thirdparty/MarkupSafe-1.0-py2-none-any.whl.ABOUT deleted file mode 100644 index c6ecc7c3..00000000 --- a/thirdparty/MarkupSafe-1.0-py2-none-any.whl.ABOUT +++ /dev/null @@ -1,19 +0,0 @@ -about_resource: MarkupSafe-1.0-py2-none-any.whl -name: MarkupSafe -version: '1' -attribute: yes -contact: armin.ronacher@active-4.com -copyright: Copyright (c) Armin Ronacher -description: | - Implements a XML/HTML/XHTML Markup safe string for Python -download_url: https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz#md5=2fcedc9284d50e577b5192e8e3578355 -homepage_url: https://pypi.python.org/pypi/MarkupSafe/ -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -notice_file: MarkupSafe-1.0.tar.gz.NOTICE -owner: Armin Ronacher -owner_url: http://lucumr.pocoo.org/about/ diff --git a/thirdparty/MarkupSafe-1.0.tar.gz b/thirdparty/MarkupSafe-1.0.tar.gz deleted file mode 100644 index 606021ae..00000000 Binary files a/thirdparty/MarkupSafe-1.0.tar.gz and /dev/null differ diff --git a/thirdparty/MarkupSafe-1.0.tar.gz.ABOUT b/thirdparty/MarkupSafe-1.0.tar.gz.ABOUT deleted file mode 100644 index 466174c5..00000000 --- a/thirdparty/MarkupSafe-1.0.tar.gz.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: MarkupSafe-1.0.tar.gz -name: MarkupSafe -version: '1' -attribute: yes -checksum_md5: 2fcedc9284d50e577b5192e8e3578355 -checksum_sha1: 9072e80a7faa0f49805737a48f3d871eb1c48728 -contact: armin.ronacher@active-4.com -copyright: Copyright (c) Armin Ronacher -description: | - Implements a XML/HTML/XHTML Markup safe string for Python -download_url: https://pypi.python.org/packages/4d/de/32d741db316d8fdb7680822dd37001ef7a448255de9699ab4bfcbdf4172b/MarkupSafe-1.0.tar.gz#md5=2fcedc9284d50e577b5192e8e3578355 -homepage_url: https://pypi.python.org/pypi/MarkupSafe/ -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -notice_file: MarkupSafe-1.0.tar.gz.NOTICE -owner: Armin Ronacher -owner_url: http://lucumr.pocoo.org/about/ diff --git a/thirdparty/MarkupSafe-1.0.tar.gz.NOTICE b/thirdparty/MarkupSafe-1.0.tar.gz.NOTICE deleted file mode 100644 index ffee1f52..00000000 --- a/thirdparty/MarkupSafe-1.0.tar.gz.NOTICE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. \ No newline at end of file diff --git a/thirdparty/PyYAML-3.12-py2-none-any.whl b/thirdparty/PyYAML-3.12-py2-none-any.whl deleted file mode 100644 index c301d137..00000000 Binary files a/thirdparty/PyYAML-3.12-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/PyYAML-3.12-py2-none-any.whl.ABOUT b/thirdparty/PyYAML-3.12-py2-none-any.whl.ABOUT deleted file mode 100644 index ff50272b..00000000 --- a/thirdparty/PyYAML-3.12-py2-none-any.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: PyYAML-3.12-py2-none-any.whl -name: PyYAML -version: '3.12' -contact: xi@resolvent.net -copyright: Copyright (c) 2006 Kirill Simonov -description: YAML parser and emitter for Python -download_url: https://pypi.python.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz#md5=4c129761b661d181ebf7ff4eb2d79950 -homepage_url: http://pyyaml.org/wiki/PyYAML -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -owner: Kirill Simonov diff --git a/thirdparty/PyYAML-3.12-py3-none-any.whl b/thirdparty/PyYAML-3.12-py3-none-any.whl deleted file mode 100644 index 4c971df0..00000000 Binary files a/thirdparty/PyYAML-3.12-py3-none-any.whl and /dev/null differ diff --git a/thirdparty/PyYAML-3.12-py3-none-any.whl.ABOUT b/thirdparty/PyYAML-3.12-py3-none-any.whl.ABOUT deleted file mode 100644 index 7cecd29f..00000000 --- a/thirdparty/PyYAML-3.12-py3-none-any.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: PyYAML-3.12-py3-none-any.whl -name: PyYAML -version: '3.12' -contact: xi@resolvent.net -copyright: Copyright (c) 2006 Kirill Simonov -description: YAML parser and emitter for Python -download_url: https://pypi.python.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz#md5=4c129761b661d181ebf7ff4eb2d79950 -homepage_url: http://pyyaml.org/wiki/PyYAML -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -owner: Kirill Simonov diff --git a/thirdparty/PyYAML-3.12.tar.gz b/thirdparty/PyYAML-3.12.tar.gz deleted file mode 100644 index aabee39f..00000000 Binary files a/thirdparty/PyYAML-3.12.tar.gz and /dev/null differ diff --git a/thirdparty/PyYAML-3.12.tar.gz.ABOUT b/thirdparty/PyYAML-3.12.tar.gz.ABOUT deleted file mode 100644 index ffebd30c..00000000 --- a/thirdparty/PyYAML-3.12.tar.gz.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: PyYAML-3.12.tar.gz -name: PyYAML -version: '3.12' -contact: xi@resolvent.net -copyright: Copyright (c) 2006 Kirill Simonov -description: YAML parser and emitter for Python -download_url: https://pypi.python.org/packages/4a/85/db5a2df477072b2902b0eb892feb37d88ac635d36245a72a6a69b23b383a/PyYAML-3.12.tar.gz#md5=4c129761b661d181ebf7ff4eb2d79950 -homepage_url: http://pyyaml.org/wiki/PyYAML -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -owner: Kirill Simonov diff --git a/thirdparty/apache-2.0.LICENSE b/thirdparty/apache-2.0.LICENSE deleted file mode 100644 index b1fac45f..00000000 --- a/thirdparty/apache-2.0.LICENSE +++ /dev/null @@ -1,201 +0,0 @@ -Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. \ No newline at end of file diff --git a/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl b/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl deleted file mode 100644 index ecc390b4..00000000 Binary files a/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl.ABOUT b/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 0fa6747f..00000000 --- a/thirdparty/atomicwrites-1.2.1-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: atomicwrites-1.2.1-py2.py3-none-any.whl -name: atomicwrites -version: 1.2.1 -attribute: yes -checksum_md5: 7335a59c5ecc22a3a6306b2caa5ff7e7 -checksum_sha1: 91442f0ffaf4ccf3abcfc8dcae1ccc810ff3ad07 -copyright: Copyright (c) Markus Unterwaditzer -download_url: https://files.pythonhosted.org/packages/3a/9a/9d878f8d885706e2530402de6417141129a943802c084238914fa6798d97/atomicwrites-1.2.1-py2.py3-none-any.whl -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -owner: Markus Unterwaditzer diff --git a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl b/thirdparty/attrs-18.2.0-py2.py3-none-any.whl deleted file mode 100644 index a51c62aa..00000000 Binary files a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.ABOUT b/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 2ffda531..00000000 --- a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,22 +0,0 @@ -about_resource: attrs-18.2.0-py2.py3-none-any.whl -name: attrs -version: 18.2.0 -attribute: yes -checksum_md5: 75dad87564a041bc7c60eeeafefb4911 -checksum_sha1: 213cd958f00be22932ac7283d552ede1aa7db1f9 -copyright: Copyright (c) Hynek Schlawack -description: | - attrs is the Python package that will bring back the joy of writing classes by relieving you from the drudgery of implementing object protocols (aka dunder methods). - - Its main goal is to help you to write concise and correct software without slowing down your code. -download_url: https://files.pythonhosted.org/packages/3a/e1/5f9023cc983f1a628a8c2fd051ad19e76ff7b142a0faf329336f9a62a514/attrs-18.2.0-py2.py3-none-any.whl -homepage_url: http://attrs.org -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: attrs-18.2.0-py2.py3-none-any.whl.NOTICE -owner: Hynek Schlawack -owner_url: http://attrs.org diff --git a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.NOTICE b/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 9e4b2aa1..00000000 --- a/thirdparty/attrs-18.2.0-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) Hynek Schlawack - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/thirdparty/backports.csv-1.0.6-py2.py3-none-any.whl b/thirdparty/backports.csv-1.0.6-py2.py3-none-any.whl deleted file mode 100644 index 5d2dc9b1..00000000 Binary files a/thirdparty/backports.csv-1.0.6-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/backports.csv.ABOUT b/thirdparty/backports.csv.ABOUT deleted file mode 100644 index 7dee1b01..00000000 --- a/thirdparty/backports.csv.ABOUT +++ /dev/null @@ -1,13 +0,0 @@ -about_resource: backports.csv-1.0.6-py2.py3-none-any.whl -name: backports.csv -version: 1.0.6 -download_url: https://files.pythonhosted.org/packages/71/f7/5db9136de67021a6dce4eefbe50d46aa043e59ebb11c83d4ecfeb47b686e/backports.csv-1.0.6-py2.py3-none-any.whl -homepage_url: https://github.com/ryanhiebert/backports.csv -licenses: - - file: python.LICENSE -notes: | - backports.csv is a backport of the csv module from Python 3. Because of this fact, - it is released under the same license as Python. -owner: Ryan Hiebert -vcs_repository: https://github.com/ryanhiebert/backports.csv.git -vcs_tool: git diff --git a/thirdparty/backports.csv.NOTICE b/thirdparty/backports.csv.NOTICE deleted file mode 100644 index 3d3b2c3a..00000000 --- a/thirdparty/backports.csv.NOTICE +++ /dev/null @@ -1,2 +0,0 @@ -backports.csv is a backport of the csv module from Python 3. -Because of this fact, it is released under the same license as Python. diff --git a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl b/thirdparty/boolean.py-3.7-py2.py3-none-any.whl deleted file mode 100644 index 6f153e95..00000000 Binary files a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.ABOUT b/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 2380120c..00000000 --- a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,8 +0,0 @@ -about_resource: boolean.py-3.7-py2.py3-none-any.whl -download_url: https://files.pythonhosted.org/packages/dc/53/b9c4f026bac231cbf7ddc214d879c3bdb5cda9a57adbe10520deeae4f154/boolean.py-3.7-py2.py3-none-any.whl -attribute: yes -checksum_md5: 97af5906f5eaa22539a034a99c24c824 -checksum_sha1: 514c2b60f10521988063a9e6b56e3fae24b393c7 -license_expression: bsd-simplified -name: boolean.py -version: '3.7' diff --git a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.NOTICE b/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 8819ea10..00000000 --- a/thirdparty/boolean.py-3.7-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2009-2017 Sebastian Kraemer, basti.kr@gmail.com -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation and/or -other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/bsd-new.LICENSE b/thirdparty/bsd-new.LICENSE deleted file mode 100644 index 0aa688ca..00000000 --- a/thirdparty/bsd-new.LICENSE +++ /dev/null @@ -1,9 +0,0 @@ -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/bsd-simplified.LICENSE b/thirdparty/bsd-simplified.LICENSE deleted file mode 100644 index 1ed897f3..00000000 --- a/thirdparty/bsd-simplified.LICENSE +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) {{YEAR}}, {{OWNER}} -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/bumpversion-0.5.4.dev0-py2.py3-none-any.whl b/thirdparty/bumpversion-0.5.4.dev0-py2.py3-none-any.whl deleted file mode 100644 index 662ed1ef..00000000 Binary files a/thirdparty/bumpversion-0.5.4.dev0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/bumpversion.ABOUT b/thirdparty/bumpversion.ABOUT deleted file mode 100644 index b7dea22a..00000000 --- a/thirdparty/bumpversion.ABOUT +++ /dev/null @@ -1,13 +0,0 @@ -about_resource: bumpversion-0.5.4.dev0-py2.py3-none-any.whl -name: bumpversion -version: 0.5.4-dev (00d121468e7) -author: - - Filip Noetzel -copyright: Copyright (C) 2013-2014 Filip Noetzel -download_url: https://github.com/peritus/bumpversion/archive/00d121468e7e753b8c05f256ea0cfb759b805226.zip -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit diff --git a/thirdparty/bumpversion.LICENSE b/thirdparty/bumpversion.LICENSE deleted file mode 100644 index d33900c5..00000000 --- a/thirdparty/bumpversion.LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (C) 2013-2014 Filip Noetzel - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl b/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl deleted file mode 100644 index 37d13a39..00000000 Binary files a/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl.ABOUT b/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 98b40f67..00000000 --- a/thirdparty/certifi-2018.4.16-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,23 +0,0 @@ -about_resource: certifi-2018.4.16-py2.py3-none-any.whl -name: certifi -version: 2018.4.16 -checksum_md5: 8280b65d50546025140b542904e86c3b -checksum_sha1: 760c62185c36483f7f7b9db7788c699e9c735990 -contact: me@kennethreitz.com -copyright: Kenneth Reitz -description: | - Certifi is a carefully curated collection of Root Certificates for validating - the trustworthiness of SSL certificates while verifying the identity of TLS hosts. - It has been extracted from the Requests project. -download_url: https://files.pythonhosted.org/packages/7c/e6/92ad559b7192d846975fc916b65f667c7b8c3a32bea7372340bfe9a15fa5/certifi-2018.4.16-py2.py3-none-any.whl#sha256=9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 -homepage_url: https://certifi.io/en/latest/ -license_expression: mpl-2.0 -licenses: - - file: mpl-2.0.LICENSE - key: mpl-2.0 - name: Mozilla Public License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mpl-2.0 -notice_file: certifi.NOTICE -notice_url: https://certifi.io/en/latest/ -owner: Kenneth Reitz -owner_url: https://github.com/kennethreitz diff --git a/thirdparty/certifi.NOTICE b/thirdparty/certifi.NOTICE deleted file mode 100644 index c2fda9a2..00000000 --- a/thirdparty/certifi.NOTICE +++ /dev/null @@ -1,21 +0,0 @@ -This package contains a modified version of ca-bundle.crt: - -ca-bundle.crt -- Bundle of CA Root Certificates - -Certificate data from Mozilla as of: Thu Nov 3 19:04:19 2011# -This is a bundle of X.509 certificates of public Certificate Authorities -(CA). These were automatically extracted from Mozilla's root certificates -file (certdata.txt). This file can be found in the mozilla source tree: -http://mxr.mozilla.org/mozilla/source/security/nss/lib/ckfw/builtins/certdata.txt?raw=1# -It contains the certificates in PEM format and therefore -can be directly used with curl / libcurl / php_curl, or with -an Apache+mod_ssl webserver for SSL client authentication. -Just configure this file as the SSLCACertificateFile.# - -***** BEGIN LICENSE BLOCK ***** -This Source Code Form is subject to the terms of the Mozilla Public License, -v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain -one at http://mozilla.org/MPL/2.0/. - -***** END LICENSE BLOCK ***** -@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $ diff --git a/thirdparty/chardet-3.0.4-py2.py3-none-any.whl b/thirdparty/chardet-3.0.4-py2.py3-none-any.whl deleted file mode 100644 index d276977d..00000000 Binary files a/thirdparty/chardet-3.0.4-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/chardet-3.0.4-py2.py3-none-any.whl.ABOUT b/thirdparty/chardet-3.0.4-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 7cd3f95f..00000000 --- a/thirdparty/chardet-3.0.4-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,23 +0,0 @@ -about_resource: chardet-3.0.4-py2.py3-none-any.whl -name: chardet -version: 3.0.4 -checksum_md5: 0004b00caff7bb543a1d0d0bd0185a03 -checksum_sha1: 96faab7de7e9a71b37f22adb64daf2898e967e3e -copyright: | - Copyright (c) Mark Pilgrim, Netscape Communications Corporation, Dan Blanchard, - Ian Cordasco and others -description: | - Python 2/3 compatible character encoding detector. -download_url: https://pypi.python.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl#md5=0004b00caff7bb543a1d0d0bd0185a03 -homepage_url: https://github.com/chardet/chardet -license_expression: lgpl-2.1-plus -licenses: - - file: lgpl-2.1-plus.LICENSE - key: lgpl-2.1-plus - name: | - GNU Lesser General Public License 2.1 or later - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:lgpl-2.1-plus -notice_url: https://github.com/chardet/chardet/blob/3.0.4/LICENSE -owner: chardet Project -owner_url: https://github.com/chardet -vcs_repository: https://github.com/chardet/chardet.git diff --git a/thirdparty/click-6.7-py2.py3-none-any.whl b/thirdparty/click-6.7-py2.py3-none-any.whl deleted file mode 100644 index 56c7ff34..00000000 Binary files a/thirdparty/click-6.7-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/click-6.7-py2.py3-none-any.whl.ABOUT b/thirdparty/click-6.7-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index d3c0cded..00000000 --- a/thirdparty/click-6.7-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,25 +0,0 @@ -about_resource: click-6.7-py2.py3-none-any.whl -name: click -version: '6.7' -checksum_md5: 5e7a4e296b3212da2ff11017675d7a4d -checksum_sha1: 9490775172e63ba94233d4b201f0e8d78b0cd939 -contact: armin.ronacher@active-4.com -copyright: Copyright (c) 2014 by Armin Ronacher. -description: | - A simple wrapper around optparse for powerful command line utilities. -download_url: https://pypi.python.org/packages/34/c1/8806f99713ddb993c5366c362b2f908f18269f8d792aff1abfd700775a77/click-6.7-py2.py3-none-any.whl#md5=5e7a4e296b3212da2ff11017675d7a4d -homepage_url: http://click.pocoo.org/ -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -notes: | - Click uses parts of optparse written by Gregory P. Ward and maintained by the - Python software foundation. This is limited to code in the parser.py module and - is under the same license as clikc itself. -owner: Armin Ronacher -owner_url: http://lucumr.pocoo.org/about/ -vcs_repository: https://github.com/pallets/click.git -vcs_tool: git diff --git a/thirdparty/click.LICENSE b/thirdparty/click.LICENSE deleted file mode 100644 index 012b628f..00000000 --- a/thirdparty/click.LICENSE +++ /dev/null @@ -1,38 +0,0 @@ -Copyright (c) 2014 by Armin Ronacher. - -Click uses parts of optparse written by Gregory P. Ward and maintained by the -Python software foundation. This is limited to code in the parser.py -module: - -Copyright (c) 2001-2006 Gregory P. Ward. All rights reserved. -Copyright (c) 2002-2006 Python Software Foundation. All rights reserved. - -Some rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/codecov-2.0.15-py2.py3-none-any.whl b/thirdparty/codecov-2.0.15-py2.py3-none-any.whl deleted file mode 100644 index 06d57b3d..00000000 Binary files a/thirdparty/codecov-2.0.15-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/codecov.ABOUT b/thirdparty/codecov.ABOUT deleted file mode 100644 index f998e24a..00000000 --- a/thirdparty/codecov.ABOUT +++ /dev/null @@ -1,12 +0,0 @@ -about_resource: codecov-2.0.15-py2.py3-none-any.whl -name: codecov -version: 2.0.15 -copyright: Copyright (c) codecov -download_url: https://pypi.python.org/packages/8b/28/4c1950a61c3c5786f0f34d643d0d28ec832433c9a7c0bd157690d4eb1d5f/codecov-2.0.15-py2.py3-none-any.whl -license_expression: apache-2.0 -licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 -owner: codecov diff --git a/thirdparty/colorama-0.3.9-py2.py3-none-any.whl b/thirdparty/colorama-0.3.9-py2.py3-none-any.whl deleted file mode 100644 index 29b83f06..00000000 Binary files a/thirdparty/colorama-0.3.9-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/colorama.ABOUT b/thirdparty/colorama.ABOUT deleted file mode 100644 index f7e3ce93..00000000 --- a/thirdparty/colorama.ABOUT +++ /dev/null @@ -1,13 +0,0 @@ -about_resource: colorama-0.3.9-py2.py3-none-any.whl -name: colorama -version: 0.3.9 -copyright: Copyright (c) 2010 Jonathan Hartley -description: Cross-platform colored terminal text -download_url: https://pypi.python.org/packages/db/c8/7dcf9dbcb22429512708fe3a547f8b6101c0d02137acbd892505aee57adf/colorama-0.3.9-py2.py3-none-any.whl#md5=cc0c01c7b3b34d0354d813e9ab26aca3 -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Jonathan Hartley diff --git a/thirdparty/colorama.LICENSE b/thirdparty/colorama.LICENSE deleted file mode 100644 index 5f567799..00000000 --- a/thirdparty/colorama.LICENSE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2010 Jonathan Hartley -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holders, nor those of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl b/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl deleted file mode 100644 index ec5973f4..00000000 Binary files a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.ABOUT b/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index fbc50a17..00000000 --- a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,25 +0,0 @@ -about_resource: funcsigs-1.0.2-py2.py3-none-any.whl -name: funcsigs -version: 1.0.2 -attribute: yes -checksum_md5: 701d58358171f34b6d1197de2923a35a -checksum_sha1: 8a601c541be194ae91f7b70075aa67d2a678ddbd -copyright: Copyright 2013 Aaron Iles -description: | - funcsigs is a backport of the PEP 362 function signature features from Python - 3.3's inspect module. The backport is compatible with Python 2.6, 2.7 as well - as 3.3 and up. 3.2 was supported by version 0.4, but with setuptools and pip no - longer supporting 3.2, we cannot make any statement about 3.2 compatibility. -download_url: https://files.pythonhosted.org/packages/69/cb/f5be453359271714c01b9bd06126eaf2e368f1fddfff30818754b5ac2328/funcsigs-1.0.2-py2.py3-none-any.whl -homepage_url: http://funcsigs.readthedocs.org/ -license_expression: apache-2.0 -licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 -notice_file: funcsigs-1.0.2-py2.py3-none-any.whl.NOTICE -notice_url: https://github.com/testing-cabal/funcsigs/blob/master/LICENSE -owner: testing-cabal -owner_url: https://github.com/testing-cabal -track_changes: yes diff --git a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.NOTICE b/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 4906f96a..00000000 --- a/thirdparty/funcsigs-1.0.2-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2013 Aaron Iles - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. \ No newline at end of file diff --git a/thirdparty/idna-2.6-py2.py3-none-any.whl b/thirdparty/idna-2.6-py2.py3-none-any.whl deleted file mode 100644 index 11cb4140..00000000 Binary files a/thirdparty/idna-2.6-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/idna-2.6-py2.py3-none-any.whl.ABOUT b/thirdparty/idna-2.6-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 919eb478..00000000 --- a/thirdparty/idna-2.6-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,32 +0,0 @@ -about_resource: idna-2.6-py2.py3-none-any.whl -name: idna -version: '2.6' -attribute: yes -checksum_md5: 875c4a7b32b4897537d5ea9247b5c79e -checksum_sha1: a75f31778ea0bbf218d7ae085f4f961d004d6ff2 -contact: http://kim.id.au/ -copyright: Copyright (c) 2013-2017, Kim Davies -description: | - Internationalized Domain Names for Python (IDNA 2008 and UTS #46) -download_url: https://pypi.python.org/packages/27/cc/6dd9a3869f15c2edfab863b992838277279ce92663d334df9ecf5106f5c6/idna-2.6-py2.py3-none-any.whl#md5=875c4a7b32b4897537d5ea9247b5c79e -homepage_url: https://github.com/kjd/idna -license_expression: bsd-new AND python AND unicode -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python - - file: unicode.LICENSE - key: unicode - name: Unicode Inc License Agreement - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:unicode -notice_file: idna-2.6-py2.py3-none-any.whl.NOTICE -notice_url: https://github.com/kjd/idna/blob/v2.6/LICENSE.rst -owner: Kim Davies -owner_url: https://github.com/kjd -track_changes: yes -vcs_repository: https://github.com/kjd/idna diff --git a/thirdparty/idna-2.6-py2.py3-none-any.whl.NOTICE b/thirdparty/idna-2.6-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 45478d85..00000000 --- a/thirdparty/idna-2.6-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,28 +0,0 @@ -Copyright (c) 2013-2017, Kim Davies. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the name of the copyright holder nor the names of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. - THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Portions of the codec implementation and unit tests are derived from the Python standard library, which carries the Python Software Foundation License: - - Copyright (c) 2001-2014 Python Software Foundation; All Rights Reserved - -Portions of the unit tests are derived from the Unicode standard, which is subject to the Unicode, Inc. License Agreement: - - Copyright (c) 1991-2014 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in . - - Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that - - (a) this copyright and permission notice appear with all copies of the Data Files or Software, - - (b) this copyright and permission notice appear in associated documentation, and - - (c) there is clear notice in each modified Data File or in the Software as well as in the documentation associated with the Data File(s) or Software that the data or software has been modified. - - THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. - - Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder. \ No newline at end of file diff --git a/thirdparty/ipaddress-1.0.16-py27-none-any.whl b/thirdparty/ipaddress-1.0.16-py27-none-any.whl deleted file mode 100644 index f1ce0dff..00000000 Binary files a/thirdparty/ipaddress-1.0.16-py27-none-any.whl and /dev/null differ diff --git a/thirdparty/ipaddress.ABOUT b/thirdparty/ipaddress.ABOUT deleted file mode 100644 index 3341e043..00000000 --- a/thirdparty/ipaddress.ABOUT +++ /dev/null @@ -1,14 +0,0 @@ -about_resource: ipaddress-1.0.16-py27-none-any.whl -name: ipaddress -version: 1.0.16 -copyright: Copyright (c) Google Inc. and others. -download_url: https://pypi.python.org/packages/cd/c5/bd44885274379121507870d4abfe7ba908326cf7bfd50a48d9d6ae091c0d/ipaddress-1.0.16.tar.gz#md5=1e27b62aa20f5b6fc200b2bdbf0d0847 -license_expression: python -licenses: - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python -owner: Philipp Hagemeister -vcs_repository: https://github.com/phihag/ipaddress.git -vcs_tool: git diff --git a/thirdparty/lgpl-2.1-plus.LICENSE b/thirdparty/lgpl-2.1-plus.LICENSE deleted file mode 100644 index fd0cb0e8..00000000 --- a/thirdparty/lgpl-2.1-plus.LICENSE +++ /dev/null @@ -1,325 +0,0 @@ -GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be introduced by others. - - Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those libraries into non-free programs. - - When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating system. - - Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must - be optional: if the application does not supply it, the square root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in these notices. - - Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies the executable. - - It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the Sections above. - - b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is -implemented by public license practices. Many people have made generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! \ No newline at end of file diff --git a/thirdparty/lgpl-2.1.LICENSE b/thirdparty/lgpl-2.1.LICENSE deleted file mode 100644 index fd0cb0e8..00000000 --- a/thirdparty/lgpl-2.1.LICENSE +++ /dev/null @@ -1,325 +0,0 @@ -GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be introduced by others. - - Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those libraries into non-free programs. - - When a program is linked with a library, whether statically or using a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating system. - - Although the Lesser General Public License is Less protective of the users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any application-supplied function or table used by this function must - be optional: if the application does not supply it, the square root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in these notices. - - Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies the executable. - - It may happen that this requirement contradicts the license restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the Sections above. - - b) Give prominent notice with the combined library of the fact that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the integrity of the free software distribution system which is -implemented by public license practices. Many people have made generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot impose that choice. - -This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! \ No newline at end of file diff --git a/thirdparty/license_expression-1.2-py2.py3-none-any.whl b/thirdparty/license_expression-1.2-py2.py3-none-any.whl deleted file mode 100644 index ab70af70..00000000 Binary files a/thirdparty/license_expression-1.2-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/license_expression-1.2-py2.py3-none-any.whl.ABOUT b/thirdparty/license_expression-1.2-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 5d869788..00000000 --- a/thirdparty/license_expression-1.2-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,19 +0,0 @@ -about_resource: license_expression-1.2-py2.py3-none-any.whl -attribute: true -checksum_md5: dc7bf41e6e232cabdc36f6c206c55710 -checksum_sha1: 942765d9af237b1bfee363c64465013b8690428d -contact: http://www.nexb.com/contactus.html -copyright: Copyright (c) 2017 nexB Inc. and others. -description: Utility library to parse, normalize and compare License expressions for - Python using a boolean logic engine. For expressions using SPDX or any other license - id scheme. -download_url: https://files.pythonhosted.org/packages/16/5d/a1f7fb7bd4f2bf1f8e065d40334d3bb8115f904ca3f86c9d0800281c9941/license_expression-1.2-py2.py3-none-any.whl -homepage_url: https://github.com/nexB/license-expression -license_expression: apache-2.0 -license_file: apache-2.0.LICENSE -name: license-expression -notice_file: license_expression-1.2-py2.py3-none-any.whl.NOTICE -notice_url: https://github.com/nexB/license-expression/blob/master/NOTICE -owner: nexB -owner_url: http://www.nexb.com/ -version: '1.2' diff --git a/thirdparty/license_expression-1.2-py2.py3-none-any.whl.NOTICE b/thirdparty/license_expression-1.2-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 8e06419e..00000000 --- a/thirdparty/license_expression-1.2-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,26 +0,0 @@ -Software license -================ - -license-expression is a free software tool from nexB Inc. and others. -Visit https://github.com/nexB/license-expression for support and download. - -Copyright (c) 2016 nexB Inc. and others. All rights reserved. -http://nexb.com and http://aboutcode.org - -This software is licensed under the Apache License version 2.0. - -You may not use this software except in compliance with the License. -You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -Third-party software licenses -============================= - -license-expression embeds third-party free and open source software packages under -various licenses. The origin and license of these packages is documented by .ABOUT -files. The corresponding source code for pre-compiled third-party software is -available in the thirdparty directory. \ No newline at end of file diff --git a/thirdparty/mock-2.0.0-py2.py3-none-any.whl b/thirdparty/mock-2.0.0-py2.py3-none-any.whl deleted file mode 100644 index 25c9874d..00000000 Binary files a/thirdparty/mock-2.0.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/mock-2.0.0-py2.py3-none-any.whl.ABOUT b/thirdparty/mock-2.0.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 42df4107..00000000 --- a/thirdparty/mock-2.0.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,19 +0,0 @@ -about_resource: mock-2.0.0-py2.py3-none-any.whl -name: mock -version: 2.0.0 -contact: http://lists.idyll.org/listinfo/testing-in-python -copyright: | - Copyright (c) 2003-2013, Michael Foord & the mock team -description: | - mock is a library for testing in Python. It allows you to replace parts of your - system under test with mock objects and make assertions about how they have been - used. -download_url: https://pypi.python.org/packages/source/m/mock/mock-2.0.0.tar.gz -homepage_url: https://github.com/testing-cabal/mock -license_expression: bsd-simplified -licenses: - - file: bsd-simplified.LICENSE - key: bsd-simplified - name: BSD-2-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-simplified -owner: Testing Cabal diff --git a/thirdparty/mock-2.0.0-py2.py3-none-any.whl.LICENSE b/thirdparty/mock-2.0.0-py2.py3-none-any.whl.LICENSE deleted file mode 100644 index f42435c1..00000000 --- a/thirdparty/mock-2.0.0-py2.py3-none-any.whl.LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2003-2013, Michael Foord & the mock team -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/thirdparty/more_itertools-4.2.0-py2-none-any.whl b/thirdparty/more_itertools-4.2.0-py2-none-any.whl deleted file mode 100644 index 5a31af74..00000000 Binary files a/thirdparty/more_itertools-4.2.0-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/more_itertools-4.2.0-py2-none-any.whl.ABOUT b/thirdparty/more_itertools-4.2.0-py2-none-any.whl.ABOUT deleted file mode 100644 index 4da3e8a0..00000000 --- a/thirdparty/more_itertools-4.2.0-py2-none-any.whl.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: more_itertools-4.2.0-py2-none-any.whl -name: more_itertools -version: 4.2.0 -attribute: yes -checksum_md5: 89 e95ea35f539a8e82c9a7b6156e4de1 -checksum_sha1: cdd72ea299f10dfec0e903d5a70d64a011f67300 -contact: grinch@grinchcentral.com -copyright: Copyright (c) Erik Rose -description: | - More routines for operating on iterables, beyond itertools -download_url: https://files.pythonhosted.org/packages/9e/92/d05d8679c3bcaa263169aa47de660080df36d35697855515745657c1ba78/more_itertools-4.2.0-py2-none-any.whl -homepage_url: https://github.com/erikrose/more-itertools -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -owner: Erik Rose -owner_url: http://www.grinchcentral.com/ -vcs_repository: https://github.com/erikrose/more-itertools diff --git a/thirdparty/more_itertools-4.2.0-py3-none-any.whl b/thirdparty/more_itertools-4.2.0-py3-none-any.whl deleted file mode 100644 index 0ac0745b..00000000 Binary files a/thirdparty/more_itertools-4.2.0-py3-none-any.whl and /dev/null differ diff --git a/thirdparty/more_itertools-4.2.0-py3-none-any.whl.ABOUT b/thirdparty/more_itertools-4.2.0-py3-none-any.whl.ABOUT deleted file mode 100644 index 07cd1adb..00000000 --- a/thirdparty/more_itertools-4.2.0-py3-none-any.whl.ABOUT +++ /dev/null @@ -1,23 +0,0 @@ -about_resource: more_itertools-4.2.0-py3-none-any.whl -name: more_itertools -version: 4.2.0 -attribute: yes -checksum_md5: 6c3bac88c1996a46a4edc7631883098a -checksum_sha1: 94c95c3b9e2d638284a401f222c455cac6fcf1e4 -contact: grinch@grinchcentral.com -copyright: Copyright (c) Erik Rose -description: | - More routines for operating on iterables, beyond itertools -download_url: https://files.pythonhosted.org/packages/85/40/90c3b0393e12b9827381004224de8814686e3d7182f9d4182477f600826d/more_itertools-4.2.0-py3-none-any.whl -homepage_url: https://github.com/erikrose/more-itertools -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: more_itertools-4.2.0-py3-none-any.whl.NOTICE -notice_url: https://github.com/erikrose/more-itertools/blob/4.2.0/LICENSE -owner: Erik Rose -owner_url: http://www.grinchcentral.com/ -vcs_repository: https://github.com/erikrose/more-itertools diff --git a/thirdparty/more_itertools-4.2.0-py3-none-any.whl.NOTICE b/thirdparty/more_itertools-4.2.0-py3-none-any.whl.NOTICE deleted file mode 100644 index 0a523bec..00000000 --- a/thirdparty/more_itertools-4.2.0-py3-none-any.whl.NOTICE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2012 Erik Rose - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/thirdparty/mpl-2.0.LICENSE b/thirdparty/mpl-2.0.LICENSE deleted file mode 100644 index c7fd5b37..00000000 --- a/thirdparty/mpl-2.0.LICENSE +++ /dev/null @@ -1,268 +0,0 @@ -Mozilla Public License Version 2.0 -================================== - -1. Definitions --------------- - -1.1. "Contributor" means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. - -1.2. "Contributor Version" means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor's Contribution. - -1.3. "Contribution" means Covered Software of a particular Contributor. - -1.4. "Covered Software" means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code - Form, and Modifications of such Source Code Form, in each case including portions thereof. - -1.5. "Incompatible With Secondary Licenses" means - - (a) that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or - - (b) that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the - terms of a Secondary License. - -1.6. "Executable Form" means any form of the work other than Source Code Form. - -1.7. "Larger Work" means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. - -1.8. "License" means this document. - -1.9. "Licensable" means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and - all of the rights conveyed by this License. - -1.10. "Modifications" means any of the following: - - (a) any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or - - (b) any new file in Source Code Form that contains any Covered Software. - -1.11. "Patent Claims" of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such - Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having - made, import, or transfer of either its Contributions or its Contributor Version. - -1.12. "Secondary License" means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General - Public License, Version 3.0, or any later versions of those licenses. - -1.13. "Source Code Form" means the form of the work preferred for making modifications. - -1.14. "You" (or "Your") means an individual or a legal entity exercising rights under this License. For legal entities, "You" includes any entity that - controls, is controlled by, or is under common control with You. For purposes of this definition, "control" means (a) the power, direct - or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than - fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. - -2. License Grants and Conditions --------------------------------- - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: - -(a) under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, - modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or - as part of a Larger Work; and - -(b) under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under this License. No additional rights or licenses will be implied from the -distribution or licensing of Covered Software under this License. Notwithstanding Section 2.1(b) above, no patent license is granted by a -Contributor: - -(a) for any code that a Contributor has removed from Covered Software; or - -(b) for infringements caused by: (i) Your and any other third party's modifications of Covered Software, or (ii) the combination of its - Contributions with other software (except as part of its Contributor Version); or - -(c) under Patent Claims infringed by Covered Software in the absence of its Contributions. - -This License does not grant any rights in the trademarks, service marks, or logos of any Contributor (except as may be necessary to comply with -the notice requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this -License (see Section 10.2) or under the terms of a Secondary License (if permitted under the terms of Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights -to grant the rights to its Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other -equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. - -3. Responsibilities -------------------- - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under -the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this -License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients' rights in the Source Code Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -(a) such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients of - the Executable Form how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more - than the cost of distribution to the recipient; and - -(b) You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the - license for the Executable Form does not attempt to limit or alter the recipients' rights in the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for -the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the -Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software -under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered -Software under the terms of either this License or such Secondary License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices (including copyright notices, patent notices, disclaimers of warranty, -or limitations of liability) contained within the Source Code Form of the Covered Software, except that You may alter any license notices to -the extent required to remedy known factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered -Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. You must make it absolutely clear that any -such warranty, support, indemnity, or liability obligation is offered by You alone, and You hereby agree to indemnify every Contributor for any -liability incurred by such Contributor as a result of warranty, support, indemnity or liability terms You offer. You may include additional -disclaimers of warranty and limitations of liability specific to any jurisdiction. - -4. Inability to Comply Due to Statute or Regulation ---------------------------------------------------- - -If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to -statute, judicial order, or regulation then You must: (a) comply with the terms of this License to the maximum extent possible; and (b) -describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered -Software under this License. Except to the extent prohibited by statute or regulation, such description must be sufficiently detailed for a -recipient of ordinary skill to be able to understand it. - -5. Termination --------------- - -5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become -compliant, then the rights granted under this License from a particular Contributor are reinstated (a) provisionally, unless and until such -Contributor explicitly and finally terminates Your grants, and (b) on an ongoing basis, if such Contributor fails to notify You of the -non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular -Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the -first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after -Your receipt of the notice. - -5.2. If You initiate litigation against any entity by asserting a patent infringement claim (excluding declaratory judgment actions, -counter-claims, and cross-claims) alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to -You by any and all Contributors for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user license agreements (excluding distributors and resellers) which -have been validly granted by You or Your distributors under this License prior to termination shall survive termination. - -************************************************************************ -* * -* 6. Disclaimer of Warranty * -* ------------------------- * -* * -* Covered Software is provided under this License on an "as is" * -* basis, without warranty of any kind, either expressed, implied, or * -* statutory, including, without limitation, warranties that the * -* Covered Software is free of defects, merchantable, fit for a * -* particular purpose or non-infringing. The entire risk as to the * -* quality and performance of the Covered Software is with You. * -* Should any Covered Software prove defective in any respect, You * -* (not any Contributor) assume the cost of any necessary servicing, * -* repair, or correction. This disclaimer of warranty constitutes an * -* essential part of this License. No use of any Covered Software is * -* authorized under this License except under this disclaimer. * -* * -************************************************************************ - -************************************************************************ -* * -* 7. Limitation of Liability * -* -------------------------- * -* * -* Under no circumstances and under no legal theory, whether tort * -* (including negligence), contract, or otherwise, shall any * -* Contributor, or anyone who distributes Covered Software as * -* permitted above, be liable to You for any direct, indirect, * -* special, incidental, or consequential damages of any character * -* including, without limitation, damages for lost profits, loss of * -* goodwill, work stoppage, computer failure or malfunction, or any * -* and all other commercial damages or losses, even if such party * -* shall have been informed of the possibility of such damages. This * -* limitation of liability shall not apply to liability for death or * -* personal injury resulting from such party's negligence to the * -* extent applicable law prohibits such limitation. Some * -* jurisdictions do not allow the exclusion or limitation of * -* incidental or consequential damages, so this exclusion and * -* limitation may not apply to You. * -* * -************************************************************************ - -8. Litigation -------------- - -Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal -place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. -Nothing in this Section shall prevent a party's ability to bring cross-claims or counter-claims. - -9. Miscellaneous ----------------- - -This License represents the complete agreement concerning the subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides -that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. - -10. Versions of the License ---------------------------- - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or -publish new versions of this License. Each version will be given a distinguishing version number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, -or under the terms of any subsequent version published by the license steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to create a new license for such software, you may create and use a -modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that -such modified license differs from this License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - -If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the -notice described in Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice -------------------------------------------- - - This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then You may include the notice in a location (such as a LICENSE -file in a relevant directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - "Incompatible With Secondary Licenses" Notice ---------------------------------------------------------- - - This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. \ No newline at end of file diff --git a/thirdparty/nltk-3.2-py2-none-any.whl b/thirdparty/nltk-3.2-py2-none-any.whl deleted file mode 100644 index bb9e8527..00000000 Binary files a/thirdparty/nltk-3.2-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/nltk.ABOUT b/thirdparty/nltk.ABOUT deleted file mode 100644 index 0d37a00e..00000000 --- a/thirdparty/nltk.ABOUT +++ /dev/null @@ -1,11 +0,0 @@ -about_resource: nltk-3.2-py2-none-any.whl -name: NLTK -version: '3.2' -download_url: https://pypi.python.org/packages/source/n/nltk/nltk-3.2.tar.gz -homepage_url: http://www.nltk.org/ -license_expression: apache-2.0 -licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 diff --git a/thirdparty/other-permissive.LICENSE b/thirdparty/other-permissive.LICENSE deleted file mode 100644 index 62c219c3..00000000 --- a/thirdparty/other-permissive.LICENSE +++ /dev/null @@ -1 +0,0 @@ -This component contains multiple third-party subcomponents licensed under permissive licenses in the style of MIT, BSD, X11, and/or Apache. \ No newline at end of file diff --git a/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl b/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl deleted file mode 100644 index dd71bea8..00000000 Binary files a/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl.ABOUT b/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 3c573725..00000000 --- a/thirdparty/packageurl_python-0.7.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: packageurl_python-0.7.0-py2.py3-none-any.whl -name: packageurl-python -version: 0.7.0 -attribute: yes -checksum_md5: 00cfa9bf7d6bb225d80f3d3fcdc50be7 -checksum_sha1: bde94e3e752b235b4a1aa57525989225c105b9d0 -copyright: Copyright (c) The purl authors -download_url: https://files.pythonhosted.org/packages/7c/48/c4e3107a8071a399a1437cc7dc19166009849bc4bb66c9553935ea9db860/packageurl_python-0.7.0-py2.py3-none-any.whl -homepage_url: https://github.com/package-url/packageurl-python -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit diff --git a/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl b/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl deleted file mode 100644 index 0cd52d85..00000000 Binary files a/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl.ABOUT b/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index cc42c737..00000000 --- a/thirdparty/pathlib2-2.3.2-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,22 +0,0 @@ -about_resource: pathlib2-2.3.2-py2.py3-none-any.whl -name: pathlib2 -version: 2.3.2 -attribute: yes -checksum_md5: 337ce92ec14734203665fca8c60d06f9 -checksum_sha1: ff29acd4cb59424e631179c2279d6fd9ac4a4566 -copyright: | - Copyright (c) 2014-2017 Matthias C. M. Troffaes Copyright (c) 2012-2014 Antoine - Pitrou and contributors -description: | - The goal of pathlib2 is to provide a backport of standard pathlib module which - tracks the standard library module, so all the newest features of the standard - pathlib can be used also on older Python versions. -download_url: https://files.pythonhosted.org/packages/66/a7/9f8d84f31728d78beade9b1271ccbfb290c41c1e4dc13dbd4997ad594dcd/pathlib2-2.3.2-py2.py3-none-any.whl -homepage_url: https://github.com/mcmtroffaes/pathlib2/ -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -vcs_repository: https://github.com/mcmtroffaes/pathlib2 diff --git a/thirdparty/pbr-4.1.0-py2.py3-none-any.whl b/thirdparty/pbr-4.1.0-py2.py3-none-any.whl deleted file mode 100644 index e7966341..00000000 Binary files a/thirdparty/pbr-4.1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/pbr-4.1.0-py2.py3-none-any.whl.ABOUT b/thirdparty/pbr-4.1.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index b65b19f0..00000000 --- a/thirdparty/pbr-4.1.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,16 +0,0 @@ -about_resource: pbr-4.1.0-py2.py3-none-any.whl -name: pbr -version: 4.1.0 -checksum_md5: 6fa3427a1cd28503459b2c6be97e3d90 -checksum_sha1: a3945f6b1de1d43df0533b7b0aecf80ae6877ee3 -copyright: | - Copyright 2011 OpenStack Foundation\r\nCopyright 2012-2013 Hewlett-Packard - Development Company, L.P. -download_url: https://files.pythonhosted.org/packages/ae/d6/2ab389a3bf5fffd03069dacaddb0cdd531594abab5119f308527c0df53e6/pbr-4.1.0-py2.py3-none-any.whl -homepage_url: https://docs.openstack.org/pbr/latest/ -license_expression: apache-2.0 -licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 diff --git a/thirdparty/pip-18.1-py2.py3-none-any.whl b/thirdparty/pip-18.1-py2.py3-none-any.whl deleted file mode 100644 index c3c146f6..00000000 Binary files a/thirdparty/pip-18.1-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/pip-18.1-py2.py3-none-any.whl.ABOUT b/thirdparty/pip-18.1-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index f5f013cd..00000000 --- a/thirdparty/pip-18.1-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,32 +0,0 @@ -about_resource: pip-18.1-py2.py3-none-any.whl -name: pip -version: '18.1' -attribute: yes -checksum_md5: 2fba06061e2274c00c67804f6ddef15e -checksum_sha1: 470a41b274f5f0f3214b97a8915a0fbde7418c8d -copyright: | - Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) -description: | - A tool for installing and managing Python packages. -download_url: https://files.pythonhosted.org/packages/c2/d7/90f34cb0d83a6c5631cf71dfe64cc1054598c843a92b400e55675cc2ac37/pip-18.1-py2.py3-none-any.whl#sha256=7909d0a0932e88ea53a7014dfd14522ffef91a464daaaf5c573343852ef98550 -homepage_url: http://www.pip-installer.org -license_expression: mit AND bsd-new AND lgpl-2.1 -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new - - file: lgpl-2.1.LICENSE - key: lgpl-2.1 - name: GNU Lesser General Public License 2.1 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:lgpl-2.1 -notice_file: pip-18.1-py2.py3-none-any.whl.NOTICE -notice_url: https://github.com/pypa/pip/blob/master/LICENSE.txt -owner: The pip developers -owner_url: http://www.pip-installer.org/en/latest/ -redistribute: yes -track_changes: yes diff --git a/thirdparty/pip-18.1-py2.py3-none-any.whl.NOTICE b/thirdparty/pip-18.1-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index a7ced4ce..00000000 --- a/thirdparty/pip-18.1-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl b/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl deleted file mode 100644 index 5d32d302..00000000 Binary files a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.ABOUT b/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 604e88aa..00000000 --- a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: pluggy-0.8.0-py2.py3-none-any.whl -name: pluggy -version: 0.8.0 -attribute: yes -checksum_md5: 15ae1293ef2c5bfda2bda23985c58b29 -checksum_sha1: ed6f9151fcba487a9bcad3b438d8752d6f96e4f7 -contact: pytest-dev@python.org -copyright: Copyright (c) Holger Krekel -description: | - Plugin and hook calling mechanisms for python -download_url: https://files.pythonhosted.org/packages/1c/e7/017c262070af41fe251401cb0d0e1b7c38f656da634cd0c15604f1f30864/pluggy-0.8.0-py2.py3-none-any.whl -homepage_url: https://github.com/pytest-dev/pluggy -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: pluggy-0.8.0-py2.py3-none-any.whl.NOTICE -owner: pytest-dev -owner_url: https://github.com/pytest-dev diff --git a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.NOTICE b/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 7fa89e83..00000000 --- a/thirdparty/pluggy-0.8.0-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 holger krekel (rather uses bitbucket/hpk42) - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/thirdparty/py-1.7.0-py2.py3-none-any.whl b/thirdparty/py-1.7.0-py2.py3-none-any.whl deleted file mode 100644 index dd9b412b..00000000 Binary files a/thirdparty/py-1.7.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/py-1.7.0-py2.py3-none-any.whl.ABOUT b/thirdparty/py-1.7.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 64583d4e..00000000 --- a/thirdparty/py-1.7.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: py-1.7.0-py2.py3-none-any.whl -name: py -version: 1.7.0 -attribute: yes -checksum_md5: 22cf37693f28856ebbb74c08d01f7c76 -checksum_sha1: e2496f9e0563ded8c70e582b7ffcff46f87b8804 -contact: pytest-dev@python.org -copyright: Copyright (c) Holger Krekel and others -description: | - library with cross-python path, ini-parsing, io, code, log facilities -download_url: https://files.pythonhosted.org/packages/3e/c7/3da685ef117d42ac8d71af525208759742dd235f8094221fdaafcd3dba8f/py-1.7.0-py2.py3-none-any.whl -homepage_url: http://pylib.readthedocs.org/ -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: py-1.7.0-py2.py3-none-any.whl.NOTICE -owner: pytest-dev -owner_url: https://github.com/pytest-dev diff --git a/thirdparty/py-1.7.0-py2.py3-none-any.whl.NOTICE b/thirdparty/py-1.7.0-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 8abfb559..00000000 --- a/thirdparty/py-1.7.0-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2004-2014 Holger Krekel and others - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. \ No newline at end of file diff --git a/thirdparty/py2-ipaddress-3.4.1.tar.gz b/thirdparty/py2-ipaddress-3.4.1.tar.gz deleted file mode 100644 index a16a586e..00000000 Binary files a/thirdparty/py2-ipaddress-3.4.1.tar.gz and /dev/null differ diff --git a/thirdparty/py2-ipaddress-3.4.1.tar.gz.ABOUT b/thirdparty/py2-ipaddress-3.4.1.tar.gz.ABOUT deleted file mode 100644 index 981c04f7..00000000 --- a/thirdparty/py2-ipaddress-3.4.1.tar.gz.ABOUT +++ /dev/null @@ -1,31 +0,0 @@ -about_resource: py2-ipaddress-3.4.1.tar.gz -name: py2-ipaddress -version: 3.4.1 -attribute: yes -checksum_md5: 47734313c841068e3d5386d048d01c3d -checksum_sha1: 26d0737c3f4d11d831ffa02df4389a428bedf205 -contact: http://www.google.com/intl/en/contact/ -copyright: Copyright (c) Google Inc. and others -description: | - Python 2.7 backport of the Python 3.3 ipaddress module. Not all 3.3 functionality - is supported, due to the lack of a distinct bytes type in Python 2.7. Nevertheless, - it is quite useful if you're stuck with Python 2.7. -download_url: https://files.pythonhosted.org/packages/06/f2/ff20f2d2fd4757be329c8ecb81e9e7fa3bec0b65445821e3a575410cf194/py2-ipaddress-3.4.1.tar.gz -homepage_url: https://bitbucket.org/kwi/py2-ipaddress -license_expression: python AND public-domain -licenses: - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python - - file: public-domain.LICENSE - key: public-domain - name: Public Domain - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain -notes: | - This is a Python 2.6 backport of the Python 3.4 ipaddress module. -notice_file: py2-ipaddress-3.4.1.tar.gz.NOTICE -owner: Google -owner_url: http://www.google.com/intl/en/about/index.html -track_changes: yes -vcs_repository: https://bitbucket.org/kwi/py2-ipaddress diff --git a/thirdparty/py2-ipaddress-3.4.1.tar.gz.NOTICE b/thirdparty/py2-ipaddress-3.4.1.tar.gz.NOTICE deleted file mode 100644 index 2915c1d3..00000000 --- a/thirdparty/py2-ipaddress-3.4.1.tar.gz.NOTICE +++ /dev/null @@ -1,10 +0,0 @@ -License -------- - -The ``ipaddress`` modules (both the original and this backport) are licensed -under the `Python Software Foundation License version 2`__. - -The modifications made for Python 2.6 compatibility are hereby released into -the public domain by the authors. - -__ https://www.python.org/download/releases/3.4.0/license \ No newline at end of file diff --git a/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl b/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl deleted file mode 100644 index 34b89a81..00000000 Binary files a/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl and /dev/null differ diff --git a/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl.ABOUT b/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl.ABOUT deleted file mode 100644 index 5aea6501..00000000 --- a/thirdparty/py2_ipaddress-3.4.1-py2-none-any.whl.ABOUT +++ /dev/null @@ -1,29 +0,0 @@ -about_resource: py2_ipaddress-3.4.1-py2-none-any.whl -name: py2-ipaddress -version: 3.4.1 -attribute: yes -contact: http://www.google.com/intl/en/contact/ -copyright: Copyright (c) Google Inc. and others -description: | - Python 2.7 backport of the Python 3.3 ipaddress module. Not all 3.3 functionality - is supported, due to the lack of a distinct bytes type in Python 2.7. Nevertheless, - it is quite useful if you're stuck with Python 2.7. -download_url: https://files.pythonhosted.org/packages/06/f2/ff20f2d2fd4757be329c8ecb81e9e7fa3bec0b65445821e3a575410cf194/py2-ipaddress-3.4.1.tar.gz -homepage_url: https://bitbucket.org/kwi/py2-ipaddress -license_expression: python AND public-domain -licenses: - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python - - file: public-domain.LICENSE - key: public-domain - name: Public Domain - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:public-domain -notes: | - This is a Python 2.6 backport of the Python 3.4 ipaddress module. -notice_file: py2-ipaddress-3.4.1.tar.gz.NOTICE -owner: Google -owner_url: http://www.google.com/intl/en/about/index.html -track_changes: yes -vcs_repository: https://bitbucket.org/kwi/py2-ipaddress diff --git a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl b/thirdparty/pytest-3.9.2-py2.py3-none-any.whl deleted file mode 100644 index 772bb4c1..00000000 Binary files a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.ABOUT b/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 0e0b3b95..00000000 --- a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: pytest-3.9.2-py2.py3-none-any.whl -name: pytest -version: 3.9.2 -attribute: yes -checksum_md5: af3e44fae34364a246d887f523e17f44 -checksum_sha1: 08477e2826f6673f1298753d7d094d3939a939b0 -contact: pytest-dev@python.org -copyright: Copyright (c) Holger Krekel and others -description: | - pytest: simple powerful testing with Python -download_url: https://files.pythonhosted.org/packages/80/2a/2995cbec008f624e1dd8ddc5350535a5d2f33c10b82c06037298e6c52bee/pytest-3.9.2-py2.py3-none-any.whl -homepage_url: http://pytest.org -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: pytest-3.9.2-py2.py3-none-any.whl.NOTICE -owner: pytest-dev -owner_url: https://github.com/pytest-dev diff --git a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.NOTICE b/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 3dba330d..00000000 --- a/thirdparty/pytest-3.9.2-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) Holger Krekel and others - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/thirdparty/python.LICENSE b/thirdparty/python.LICENSE deleted file mode 100644 index e3529741..00000000 --- a/thirdparty/python.LICENSE +++ /dev/null @@ -1,192 +0,0 @@ -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python -alone or in any derivative version, provided, however, that PSF's -License Agreement and PSF's notice of copyright, i.e., "Copyright (c) -2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights -Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - -CNRI OPEN SOURCE LICENSE AGREEMENT (for Python 1.6b1) --------------------------------------------------- - -IMPORTANT: PLEASE READ THE FOLLOWING AGREEMENT CAREFULLY. - -BY CLICKING ON "ACCEPT" WHERE INDICATED BELOW, OR BY COPYING, -INSTALLING OR OTHERWISE USING PYTHON 1.6, beta 1 SOFTWARE, YOU ARE -DEEMED TO HAVE AGREED TO THE TERMS AND CONDITIONS OF THIS LICENSE -AGREEMENT. - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6, beta 1 -software in source or binary form and its associated documentation, -as released at the www.python.org Internet site on August 4, 2000 -("Python 1.6b1"). - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a non-exclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display -publicly, prepare derivative works, distribute, and otherwise use -Python 1.6b1 alone or in any derivative version, provided, however, -that CNRIs License Agreement is retained in Python 1.6b1, alone or -in any derivative version prepared by Licensee. - -Alternately, in lieu of CNRIs License Agreement, Licensee may -substitute the following text (omitting the quotes): "Python 1.6, -beta 1, is made available subject to the terms and conditions in -CNRIs License Agreement. This Agreement may be located on the -Internet using the following unique, persistent identifier (known -as a handle): 1895.22/1011. This Agreement may also be obtained -from a proxy server on the Internet using the -URL:http://hdl.handle.net/1895.22/1011". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6b1 or any part thereof, and wants to make -the derivative work available to the public as provided herein, -then Licensee hereby agrees to indicate in any such work the nature -of the modifications made to Python 1.6b1. - -4. CNRI is making Python 1.6b1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR -FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6b1 -WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR -LOSS AS A RESULT OF USING, MODIFYING OR DISTRIBUTING PYTHON 1.6b1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY -THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of Virginia, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed -to create any relationship of agency, partnership, or joint venture -between CNRI and Licensee. This License Agreement does not grant -permission to use CNRI trademarks or trade name in a trademark -sense to endorse or promote products or services of Licensee, or -any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6b1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - -ACCEPT - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file diff --git a/thirdparty/pyyaml.LICENSE b/thirdparty/pyyaml.LICENSE deleted file mode 100644 index 050ced23..00000000 --- a/thirdparty/pyyaml.LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2006 Kirill Simonov - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/thirdparty/requests-2.18.4-py2.py3-none-any.whl b/thirdparty/requests-2.18.4-py2.py3-none-any.whl deleted file mode 100644 index cf3bdc07..00000000 Binary files a/thirdparty/requests-2.18.4-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/requests-2.18.4-py2.py3-none-any.whl.ABOUT b/thirdparty/requests-2.18.4-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index d7629daa..00000000 --- a/thirdparty/requests-2.18.4-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,22 +0,0 @@ -about_resource: requests-2.18.4-py2.py3-none-any.whl -name: Requests -version: 2.18.4 -checksum_md5: eb9be71cc41fd73a51a7c9cd1adde5de -checksum_sha1: 52ccdd6ee808bddd0c6eabc6eda79e79381266df -contact: me@kennethreitz.com -copyright: Copyright (c) 2017 Kenneth Reitz -description: | - Requests is the only Non-GMO HTTP library for Python, safe for human consumption. -download_url: https://pypi.python.org/packages/49/df/50aa1999ab9bde74656c2919d9c0c085fd2b3775fd3eca826012bef76d8c/requests-2.18.4-py2.py3-none-any.whl#md5=eb9be71cc41fd73a51a7c9cd1adde5de -homepage_url: http://python-requests.org -license_expression: apache-2.0 -licenses: - - file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:apache-2.0 -notice_file: requests.NOTICE -owner: Kenneth Reitz -owner_url: https://github.com/kennethreitz -vcs_repository: https://github.com/kennethreitz/requests.git -vcs_tool: git diff --git a/thirdparty/requests.NOTICE b/thirdparty/requests.NOTICE deleted file mode 100644 index db78ea69..00000000 --- a/thirdparty/requests.NOTICE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2017 Kenneth Reitz - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl deleted file mode 100644 index c8aa66fa..00000000 Binary files a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT b/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index ae857b3e..00000000 --- a/thirdparty/saneyaml-0.1-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,20 +0,0 @@ -about_resource: saneyaml-0.1-py2.py3-none-any.whl -attribute: true -checksum_md5: 53509e4f1ee9f6565158163a56513b7c -checksum_sha1: a66920309794ead47711473a21f1c7d003acd005 -contact: http://www.nexb.com/contactus.html -copyright: Copyright (c) nexB Inc. and others. -download_url: https://files.pythonhosted.org/packages/21/13/3d639adcd97fc22cfc4acfd23330cc26f96a85a5b992d93086684e24ac14/saneyaml-0.1-py2.py3-none-any.whl -homepage_url: https://github.com/nexB/saneyaml -license_expression: apache-2.0 -licenses: -- file: apache-2.0.LICENSE - key: apache-2.0 - name: Apache License 2.0 -name: saneyaml -owner: nexB -owner_url: http://www.nexb.com/ -package_url: pkg:pypi/saneyaml@0.1 -track_changes: true -vcs_repository: https://github.com/nexB/saneyaml -version: '0.1' diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl b/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl deleted file mode 100644 index aa9d968a..00000000 Binary files a/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl and /dev/null differ diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl.ABOUT b/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl.ABOUT deleted file mode 100644 index 043fcf66..00000000 --- a/thirdparty/scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: scandir-1.9.0-cp27-cp27m-macosx_10_11_x86_64.whl -name: scandir -version: 1.9.0 -attribute: yes -copyright: Copyright (c) Ben Hoyt -download_url: https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz -homepage_url: https://github.com/benhoyt/scandir -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Ben Hoyt -vcs_repository: https://github.com/benhoyt/scandir diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl b/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl deleted file mode 100644 index fd2f1047..00000000 Binary files a/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl and /dev/null differ diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl.ABOUT b/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl.ABOUT deleted file mode 100644 index a59249ea..00000000 --- a/thirdparty/scandir-1.9.0-cp27-cp27m-win32.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: scandir-1.9.0-cp27-cp27m-win32.whl -name: scandir -version: 1.9.0 -attribute: yes -copyright: Copyright (c) Ben Hoyt -download_url: https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz -homepage_url: https://github.com/benhoyt/scandir -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Ben Hoyt -vcs_repository: https://github.com/benhoyt/scandir diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl b/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl deleted file mode 100644 index 7c718c67..00000000 Binary files a/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl.ABOUT b/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl.ABOUT deleted file mode 100644 index 87299a22..00000000 --- a/thirdparty/scandir-1.9.0-cp27-cp27m-win_amd64.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: scandir-1.9.0-cp27-cp27m-win_amd64.whl -name: scandir -version: 1.9.0 -attribute: yes -copyright: Copyright (c) Ben Hoyt -download_url: https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz -homepage_url: https://github.com/benhoyt/scandir -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Ben Hoyt -vcs_repository: https://github.com/benhoyt/scandir diff --git a/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl b/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl deleted file mode 100644 index 02641165..00000000 Binary files a/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl and /dev/null differ diff --git a/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl.ABOUT b/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl.ABOUT deleted file mode 100644 index e9385025..00000000 --- a/thirdparty/scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: scandir-1.9.0-cp27-cp27mu-linux_x86_64.whl -name: scandir -version: 1.9.0 -attribute: yes -copyright: Copyright (c) Ben Hoyt -download_url: https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz -homepage_url: https://github.com/benhoyt/scandir -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Ben Hoyt -vcs_repository: https://github.com/benhoyt/scandir diff --git a/thirdparty/scandir-1.9.0.tar.gz b/thirdparty/scandir-1.9.0.tar.gz deleted file mode 100644 index f134aace..00000000 Binary files a/thirdparty/scandir-1.9.0.tar.gz and /dev/null differ diff --git a/thirdparty/scandir-1.9.0.tar.gz.ABOUT b/thirdparty/scandir-1.9.0.tar.gz.ABOUT deleted file mode 100644 index a26adf93..00000000 --- a/thirdparty/scandir-1.9.0.tar.gz.ABOUT +++ /dev/null @@ -1,17 +0,0 @@ -about_resource: scandir-1.9.0.tar.gz -name: scandir -version: 1.9.0 -attribute: yes -checksum_md5: 506c4cc5f38c00b301642a9cb0433910 -checksum_sha1: 64c550daec4ef70aa913e1e046b265ff9914c3e8 -copyright: Copyright (c) Ben Hoyt -download_url: https://files.pythonhosted.org/packages/16/2a/557af1181e6b4e30254d5a6163b18f5053791ca66e251e77ab08887e8fe3/scandir-1.9.0.tar.gz -homepage_url: https://github.com/benhoyt/scandir -license_expression: bsd-new -licenses: - - file: bsd-new.LICENSE - key: bsd-new - name: BSD-3-Clause - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:bsd-new -owner: Ben Hoyt -vcs_repository: https://github.com/benhoyt/scandir diff --git a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl b/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl deleted file mode 100644 index 7b1402d7..00000000 Binary files a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.ABOUT b/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 3798e8f5..00000000 --- a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,26 +0,0 @@ -about_resource: setuptools-40.4.3-py2.py3-none-any.whl -name: setuptools -version: 40.4.3 -attribute: yes -checksum_md5: dcc78bea79a5eab83e7174fb4000d804 -checksum_sha1: 40410299ab3c7b091f9dff4dd7bda7f874c4d1b6 -contact: distutils-sig@python.org -copyright: Copyright (c) Python Packaging Authority -description: | - Easily download, build, install, upgrade, and uninstall Python packages -download_url: https://files.pythonhosted.org/packages/96/06/c8ee69628191285ddddffb277bd5abdf769166e7a14b867c2a172f0175b1/setuptools-40.4.3-py2.py3-none-any.whl#sha256=ce4137d58b444bac11a31d4e0c1805c69d89e8ed4e91fde1999674ecc2f6f9ff -homepage_url: https://pypi.python.org/pypi/setuptools -license_expression: python AND mit -licenses: - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: setuptools-40.4.3-py2.py3-none-any.whl.NOTICE -owner: Python Packaging Authority -owner_url: https://www.pypa.io/en/latest/ -track_changes: yes diff --git a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.NOTICE b/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 308e3587..00000000 --- a/thirdparty/setuptools-40.4.3-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2016 Jason R Coombs - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/thirdparty/six-1.11.0-py2.py3-none-any.whl b/thirdparty/six-1.11.0-py2.py3-none-any.whl deleted file mode 100644 index 59960239..00000000 Binary files a/thirdparty/six-1.11.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/six-1.11.0-py2.py3-none-any.whl.ABOUT b/thirdparty/six-1.11.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 9c7ce413..00000000 --- a/thirdparty/six-1.11.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,25 +0,0 @@ -about_resource: six-1.11.0-py2.py3-none-any.whl -name: six -version: 1.11.0 -checksum_md5: 866ab722be6bdfed6830f3179af65468 -checksum_sha1: fa2683a24d4a7422add33400048fc375b2afe57b -contact: benjamin@python.org -copyright: | - Copyright (c) 2010-2011 Benjamin Peterson -description: | - Six is a Python 2 and 3 compatibility library. It provides utility functions for - smoothing over the differences between the Python versions with the goal of writing - Python code that is compatible on both Python versions. -download_url: https://pypi.python.org/packages/67/4b/141a581104b1f6397bfa78ac9d43d8ad29a7ca43ea90a2d863fe3056e86a/six-1.11.0-py2.py3-none-any.whl -homepage_url: http://bitbucket.org/gutworth/six -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: six.LICENSE -notice_url: https://github.com/benjaminp/six/blob/master/LICENSE -owner: Benjamin Peterson -owner_url: http://www.benjamin-peterson.org/ -vcs_repository: https://bitbucket.org/gutworth/six diff --git a/thirdparty/six.LICENSE b/thirdparty/six.LICENSE deleted file mode 100644 index 8e2cd307..00000000 --- a/thirdparty/six.LICENSE +++ /dev/null @@ -1,18 +0,0 @@ -Copyright (c) 2010-2011 Benjamin Peterson - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/thirdparty/unicode.LICENSE b/thirdparty/unicode.LICENSE deleted file mode 100644 index 6d0fbca5..00000000 --- a/thirdparty/unicode.LICENSE +++ /dev/null @@ -1,17 +0,0 @@ -UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE - - Unicode Data Files include all data files under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, and http://www.unicode.org/cldr/data/ . Unicode Software includes any source code published in the Unicode Standard or under the directories http://www.unicode.org/Public/, http://www.unicode.org/reports/, and http://www.unicode.org/cldr/data/. - - NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES ("DATA FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. - - COPYRIGHT AND PERMISSION NOTICE - - Copyright © 1991-2009 Unicode, Inc. All rights reserved. Distributed under the Terms of Use in http://www.unicode.org/copyright.html. - - Permission is hereby granted, free of charge, to any person obtaining a copy of the Unicode data files and any associated documentation (the "Data Files") or Unicode software and any associated documentation (the "Software") to deal in the Data Files or Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, and/or sell copies of the Data Files or Software, and to permit persons to whom the Data Files or Software are furnished to do so, provided that (a) the above copyright notice(s) and this permission notice appear with all copies of the Data Files or Software, (b) both the above copyright notice(s) and this permission notice appear in associated documentation, and (c) there is clear notice in each modified Data File or in the Software as well as in the documentation associated with the Data File(s) or Software that the data or software has been modified. - - THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR SOFTWARE. - - Except as contained in this notice, the name of a copyright holder shall not be used in advertising or otherwise to promote the sale, use or other dealings in these Data Files or Software without prior written authorization of the copyright holder. - - Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be registered in some jurisdictions. All other trademarks and registered trademarks mentioned herein are the property of their respective owners. diff --git a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl b/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl deleted file mode 100644 index 1a906569..00000000 Binary files a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.ABOUT b/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 18ed2aa3..00000000 --- a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,23 +0,0 @@ -about_resource: virtualenv-16.0.0-py2.py3-none-any.whl -name: virtualenv -version: 16.0.0 -attribute: yes -checksum_md5: d9ea4de7dc7830a8deec1a9f59de0a8a -checksum_sha1: dc15dc5965f4ecff4f8a5cbdc7fa3af38d2f03a9 -copyright: | - Copyright (c) 2007 Ian Bicking and Contributors - Copyright (c) 2009 Ian Bicking, The Open Planning Project - Copyright (c) The virtualenv developers -description: | - virtualenv is a tool to create isolated Python environments. -download_url: https://files.pythonhosted.org/packages/b6/30/96a02b2287098b23b875bc8c2f58071c35d2efe84f747b64d523721dc2b5/virtualenv-16.0.0-py2.py3-none-any.whl -homepage_url: http://virtualenv.org/ -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: virtualenv-16.0.0-py2.py3-none-any.whl.NOTICE -owner: The virtualenv developers -owner_url: http://www.virtualenv.org/en/latest/ diff --git a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.NOTICE b/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.NOTICE deleted file mode 100644 index 08dd0f29..00000000 --- a/thirdparty/virtualenv-16.0.0-py2.py3-none-any.whl.NOTICE +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2016 The virtualenv developers - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/thirdparty/virtualenv.py b/thirdparty/virtualenv.py deleted file mode 100644 index cbf41de2..00000000 --- a/thirdparty/virtualenv.py +++ /dev/null @@ -1,2347 +0,0 @@ -#!/usr/bin/env python -"""Create a "virtual" Python installation""" - -import os -import sys - -# If we are running in a new interpreter to create a virtualenv, -# we do NOT want paths from our existing location interfering with anything, -# So we remove this file's directory from sys.path - most likely to be -# the previous interpreter's site-packages. Solves #705, #763, #779 -if os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'): - for path in sys.path[:]: - if os.path.realpath(os.path.dirname(__file__)) == os.path.realpath(path): - sys.path.remove(path) - -import base64 -import codecs -import optparse -import re -import shutil -import logging -import zlib -import errno -import glob -import distutils.spawn -import distutils.sysconfig -import struct -import subprocess -import pkgutil -import tempfile -import textwrap -from distutils.util import strtobool -from os.path import join - -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser - -__version__ = "16.0.0" -virtualenv_version = __version__ # legacy - -if sys.version_info < (2, 7): - print('ERROR: %s' % sys.exc_info()[1]) - print('ERROR: this script requires Python 2.7 or greater.') - sys.exit(101) - -try: - basestring -except NameError: - basestring = str - -py_version = 'python%s.%s' % (sys.version_info[0], sys.version_info[1]) - -is_jython = sys.platform.startswith('java') -is_pypy = hasattr(sys, 'pypy_version_info') -is_win = (sys.platform == 'win32') -is_cygwin = (sys.platform == 'cygwin') -is_darwin = (sys.platform == 'darwin') -abiflags = getattr(sys, 'abiflags', '') - -user_dir = os.path.expanduser('~') -if is_win: - default_storage_dir = os.path.join(user_dir, 'virtualenv') -else: - default_storage_dir = os.path.join(user_dir, '.virtualenv') -default_config_file = os.path.join(default_storage_dir, 'virtualenv.ini') - -if is_pypy: - expected_exe = 'pypy' -elif is_jython: - expected_exe = 'jython' -else: - expected_exe = 'python' - -# Return a mapping of version -> Python executable -# Only provided for Windows, where the information in the registry is used -if not is_win: - def get_installed_pythons(): - return {} -else: - try: - import winreg - except ImportError: - import _winreg as winreg - - def get_installed_pythons(): - try: - python_core = winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, - "Software\\Python\\PythonCore") - except WindowsError: - # No registered Python installations - return {} - i = 0 - versions = [] - while True: - try: - versions.append(winreg.EnumKey(python_core, i)) - i = i + 1 - except WindowsError: - break - exes = dict() - for ver in versions: - try: - path = winreg.QueryValue(python_core, "%s\\InstallPath" % ver) - except WindowsError: - continue - exes[ver] = join(path, "python.exe") - - winreg.CloseKey(python_core) - - # Add the major versions - # Sort the keys, then repeatedly update the major version entry - # Last executable (i.e., highest version) wins with this approach - for ver in sorted(exes): - exes[ver[0]] = exes[ver] - - return exes - -REQUIRED_MODULES = ['os', 'posix', 'posixpath', 'nt', 'ntpath', 'genericpath', - 'fnmatch', 'locale', 'encodings', 'codecs', - 'stat', 'UserDict', 'readline', 'copy_reg', 'types', - 're', 'sre', 'sre_parse', 'sre_constants', 'sre_compile', - 'zlib'] - -REQUIRED_FILES = ['lib-dynload', 'config'] - -majver, minver = sys.version_info[:2] -if majver == 2: - if minver >= 6: - REQUIRED_MODULES.extend(['warnings', 'linecache', '_abcoll', 'abc']) - if minver >= 7: - REQUIRED_MODULES.extend(['_weakrefset']) -elif majver == 3: - # Some extra modules are needed for Python 3, but different ones - # for different versions. - REQUIRED_MODULES.extend([ - '_abcoll', 'warnings', 'linecache', 'abc', 'io', '_weakrefset', - 'copyreg', 'tempfile', 'random', '__future__', 'collections', - 'keyword', 'tarfile', 'shutil', 'struct', 'copy', 'tokenize', - 'token', 'functools', 'heapq', 'bisect', 'weakref', 'reprlib' - ]) - if minver >= 2: - REQUIRED_FILES[-1] = 'config-%s' % majver - if minver >= 3: - import sysconfig - platdir = sysconfig.get_config_var('PLATDIR') - REQUIRED_FILES.append(platdir) - REQUIRED_MODULES.extend([ - 'base64', '_dummy_thread', 'hashlib', 'hmac', - 'imp', 'importlib', 'rlcompleter' - ]) - if minver >= 4: - REQUIRED_MODULES.extend([ - 'operator', - '_collections_abc', - '_bootlocale', - ]) - if minver >= 6: - REQUIRED_MODULES.extend(['enum']) - -if is_pypy: - # these are needed to correctly display the exceptions that may happen - # during the bootstrap - REQUIRED_MODULES.extend(['traceback', 'linecache']) - - if majver == 3: - # _functools is needed to import locale during stdio initialization and - # needs to be copied on PyPy because it's not built in - REQUIRED_MODULES.append('_functools') - - -class Logger(object): - - """ - Logging object for use in command-line script. Allows ranges of - levels, to avoid some redundancy of displayed information. - """ - - DEBUG = logging.DEBUG - INFO = logging.INFO - NOTIFY = (logging.INFO+logging.WARN)/2 - WARN = WARNING = logging.WARN - ERROR = logging.ERROR - FATAL = logging.FATAL - - LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL] - - def __init__(self, consumers): - self.consumers = consumers - self.indent = 0 - self.in_progress = None - self.in_progress_hanging = False - - def debug(self, msg, *args, **kw): - self.log(self.DEBUG, msg, *args, **kw) - - def info(self, msg, *args, **kw): - self.log(self.INFO, msg, *args, **kw) - - def notify(self, msg, *args, **kw): - self.log(self.NOTIFY, msg, *args, **kw) - - def warn(self, msg, *args, **kw): - self.log(self.WARN, msg, *args, **kw) - - def error(self, msg, *args, **kw): - self.log(self.ERROR, msg, *args, **kw) - - def fatal(self, msg, *args, **kw): - self.log(self.FATAL, msg, *args, **kw) - - def log(self, level, msg, *args, **kw): - if args: - if kw: - raise TypeError( - "You may give positional or keyword arguments, not both") - args = args or kw - rendered = None - for consumer_level, consumer in self.consumers: - if self.level_matches(level, consumer_level): - if (self.in_progress_hanging - and consumer in (sys.stdout, sys.stderr)): - self.in_progress_hanging = False - sys.stdout.write('\n') - sys.stdout.flush() - if rendered is None: - if args: - rendered = msg % args - else: - rendered = msg - rendered = ' '*self.indent + rendered - if hasattr(consumer, 'write'): - consumer.write(rendered+'\n') - else: - consumer(rendered) - - def start_progress(self, msg): - assert not self.in_progress, ( - "Tried to start_progress(%r) while in_progress %r" - % (msg, self.in_progress)) - if self.level_matches(self.NOTIFY, self._stdout_level()): - sys.stdout.write(msg) - sys.stdout.flush() - self.in_progress_hanging = True - else: - self.in_progress_hanging = False - self.in_progress = msg - - def end_progress(self, msg='done.'): - assert self.in_progress, ( - "Tried to end_progress without start_progress") - if self.stdout_level_matches(self.NOTIFY): - if not self.in_progress_hanging: - # Some message has been printed out since start_progress - sys.stdout.write('...' + self.in_progress + msg + '\n') - sys.stdout.flush() - else: - sys.stdout.write(msg + '\n') - sys.stdout.flush() - self.in_progress = None - self.in_progress_hanging = False - - def show_progress(self): - """If we are in a progress scope, and no log messages have been - shown, write out another '.'""" - if self.in_progress_hanging: - sys.stdout.write('.') - sys.stdout.flush() - - def stdout_level_matches(self, level): - """Returns true if a message at this level will go to stdout""" - return self.level_matches(level, self._stdout_level()) - - def _stdout_level(self): - """Returns the level that stdout runs at""" - for level, consumer in self.consumers: - if consumer is sys.stdout: - return level - return self.FATAL - - def level_matches(self, level, consumer_level): - """ - >>> l = Logger([]) - >>> l.level_matches(3, 4) - False - >>> l.level_matches(3, 2) - True - >>> l.level_matches(slice(None, 3), 3) - False - >>> l.level_matches(slice(None, 3), 2) - True - >>> l.level_matches(slice(1, 3), 1) - True - >>> l.level_matches(slice(2, 3), 1) - False - """ - if isinstance(level, slice): - start, stop = level.start, level.stop - if start is not None and start > consumer_level: - return False - if stop is not None and stop <= consumer_level: - return False - return True - else: - return level >= consumer_level - - @classmethod - def level_for_integer(cls, level): - levels = cls.LEVELS - if level < 0: - return levels[0] - if level >= len(levels): - return levels[-1] - return levels[level] - -# create a silent logger just to prevent this from being undefined -# will be overridden with requested verbosity main() is called. -logger = Logger([(Logger.LEVELS[-1], sys.stdout)]) - -def mkdir(path): - if not os.path.exists(path): - logger.info('Creating %s', path) - os.makedirs(path) - else: - logger.info('Directory %s already exists', path) - -def copyfileordir(src, dest, symlink=True): - if os.path.isdir(src): - shutil.copytree(src, dest, symlink) - else: - shutil.copy2(src, dest) - -def copyfile(src, dest, symlink=True): - if not os.path.exists(src): - # Some bad symlink in the src - logger.warn('Cannot find file %s (bad symlink)', src) - return - if os.path.exists(dest): - logger.debug('File %s already exists', dest) - return - if not os.path.exists(os.path.dirname(dest)): - logger.info('Creating parent directories for %s', os.path.dirname(dest)) - os.makedirs(os.path.dirname(dest)) - if not os.path.islink(src): - srcpath = os.path.abspath(src) - else: - srcpath = os.readlink(src) - if symlink and hasattr(os, 'symlink') and not is_win: - logger.info('Symlinking %s', dest) - try: - os.symlink(srcpath, dest) - except (OSError, NotImplementedError): - logger.info('Symlinking failed, copying to %s', dest) - copyfileordir(src, dest, symlink) - else: - logger.info('Copying to %s', dest) - copyfileordir(src, dest, symlink) - -def writefile(dest, content, overwrite=True): - if not os.path.exists(dest): - logger.info('Writing %s', dest) - with open(dest, 'wb') as f: - f.write(content.encode('utf-8')) - return - else: - with open(dest, 'rb') as f: - c = f.read() - if c != content.encode("utf-8"): - if not overwrite: - logger.notify('File %s exists with different content; not overwriting', dest) - return - logger.notify('Overwriting %s with new content', dest) - with open(dest, 'wb') as f: - f.write(content.encode('utf-8')) - else: - logger.info('Content %s already in place', dest) - -def rmtree(dir): - if os.path.exists(dir): - logger.notify('Deleting tree %s', dir) - shutil.rmtree(dir) - else: - logger.info('Do not need to delete %s; already gone', dir) - -def make_exe(fn): - if hasattr(os, 'chmod'): - oldmode = os.stat(fn).st_mode & 0xFFF # 0o7777 - newmode = (oldmode | 0x16D) & 0xFFF # 0o555, 0o7777 - os.chmod(fn, newmode) - logger.info('Changed mode of %s to %s', fn, oct(newmode)) - -def _find_file(filename, dirs): - for dir in reversed(dirs): - files = glob.glob(os.path.join(dir, filename)) - if files and os.path.isfile(files[0]): - return True, files[0] - return False, filename - -def file_search_dirs(): - here = os.path.dirname(os.path.abspath(__file__)) - dirs = [here, join(here, 'virtualenv_support')] - if os.path.splitext(os.path.dirname(__file__))[0] != 'virtualenv': - # Probably some boot script; just in case virtualenv is installed... - try: - import virtualenv - except ImportError: - pass - else: - dirs.append(os.path.join( - os.path.dirname(virtualenv.__file__), 'virtualenv_support')) - return [d for d in dirs if os.path.isdir(d)] - - -class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter): - """ - Custom help formatter for use in ConfigOptionParser that updates - the defaults before expanding them, allowing them to show up correctly - in the help listing - """ - def expand_default(self, option): - if self.parser is not None: - self.parser.update_defaults(self.parser.defaults) - return optparse.IndentedHelpFormatter.expand_default(self, option) - - -class ConfigOptionParser(optparse.OptionParser): - """ - Custom option parser which updates its defaults by checking the - configuration files and environmental variables - """ - def __init__(self, *args, **kwargs): - self.config = ConfigParser.RawConfigParser() - self.files = self.get_config_files() - self.config.read(self.files) - optparse.OptionParser.__init__(self, *args, **kwargs) - - def get_config_files(self): - config_file = os.environ.get('VIRTUALENV_CONFIG_FILE', False) - if config_file and os.path.exists(config_file): - return [config_file] - return [default_config_file] - - def update_defaults(self, defaults): - """ - Updates the given defaults with values from the config files and - the environ. Does a little special handling for certain types of - options (lists). - """ - # Then go and look for the other sources of configuration: - config = {} - # 1. config files - config.update(dict(self.get_config_section('virtualenv'))) - # 2. environmental variables - config.update(dict(self.get_environ_vars())) - # Then set the options with those values - for key, val in config.items(): - key = key.replace('_', '-') - if not key.startswith('--'): - key = '--%s' % key # only prefer long opts - option = self.get_option(key) - if option is not None: - # ignore empty values - if not val: - continue - # handle multiline configs - if option.action == 'append': - val = val.split() - else: - option.nargs = 1 - if option.action == 'store_false': - val = not strtobool(val) - elif option.action in ('store_true', 'count'): - val = strtobool(val) - try: - val = option.convert_value(key, val) - except optparse.OptionValueError: - e = sys.exc_info()[1] - print("An error occurred during configuration: %s" % e) - sys.exit(3) - defaults[option.dest] = val - return defaults - - def get_config_section(self, name): - """ - Get a section of a configuration - """ - if self.config.has_section(name): - return self.config.items(name) - return [] - - def get_environ_vars(self, prefix='VIRTUALENV_'): - """ - Returns a generator with all environmental vars with prefix VIRTUALENV - """ - for key, val in os.environ.items(): - if key.startswith(prefix): - yield (key.replace(prefix, '').lower(), val) - - def get_default_values(self): - """ - Overridding to make updating the defaults after instantiation of - the option parser possible, update_defaults() does the dirty work. - """ - if not self.process_default_values: - # Old, pre-Optik 1.5 behaviour. - return optparse.Values(self.defaults) - - defaults = self.update_defaults(self.defaults.copy()) # ours - for option in self._get_all_options(): - default = defaults.get(option.dest) - if isinstance(default, basestring): - opt_str = option.get_opt_string() - defaults[option.dest] = option.check_value(opt_str, default) - return optparse.Values(defaults) - - -def main(): - parser = ConfigOptionParser( - version=virtualenv_version, - usage="%prog [OPTIONS] DEST_DIR", - formatter=UpdatingDefaultsHelpFormatter()) - - parser.add_option( - '-v', '--verbose', - action='count', - dest='verbose', - default=0, - help="Increase verbosity.") - - parser.add_option( - '-q', '--quiet', - action='count', - dest='quiet', - default=0, - help='Decrease verbosity.') - - parser.add_option( - '-p', '--python', - dest='python', - metavar='PYTHON_EXE', - help='The Python interpreter to use, e.g., --python=python3.5 will use the python3.5 ' - 'interpreter to create the new environment. The default is the interpreter that ' - 'virtualenv was installed with (%s)' % sys.executable) - - parser.add_option( - '--clear', - dest='clear', - action='store_true', - help="Clear out the non-root install and start from scratch.") - - parser.set_defaults(system_site_packages=False) - parser.add_option( - '--no-site-packages', - dest='system_site_packages', - action='store_false', - help="DEPRECATED. Retained only for backward compatibility. " - "Not having access to global site-packages is now the default behavior.") - - parser.add_option( - '--system-site-packages', - dest='system_site_packages', - action='store_true', - help="Give the virtual environment access to the global site-packages.") - - parser.add_option( - '--always-copy', - dest='symlink', - action='store_false', - default=True, - help="Always copy files rather than symlinking.") - - parser.add_option( - '--relocatable', - dest='relocatable', - action='store_true', - help='Make an EXISTING virtualenv environment relocatable. ' - 'This fixes up scripts and makes all .pth files relative.') - - parser.add_option( - '--no-setuptools', - dest='no_setuptools', - action='store_true', - help='Do not install setuptools in the new virtualenv.') - - parser.add_option( - '--no-pip', - dest='no_pip', - action='store_true', - help='Do not install pip in the new virtualenv.') - - parser.add_option( - '--no-wheel', - dest='no_wheel', - action='store_true', - help='Do not install wheel in the new virtualenv.') - - default_search_dirs = file_search_dirs() - parser.add_option( - '--extra-search-dir', - dest="search_dirs", - action="append", - metavar='DIR', - default=default_search_dirs, - help="Directory to look for setuptools/pip distributions in. " - "This option can be used multiple times.") - - parser.add_option( - "--download", - dest="download", - default=True, - action="store_true", - help="Download preinstalled packages from PyPI.", - ) - - parser.add_option( - "--no-download", - '--never-download', - dest="download", - action="store_false", - help="Do not download preinstalled packages from PyPI.", - ) - - parser.add_option( - '--prompt', - dest='prompt', - help='Provides an alternative prompt prefix for this environment.') - - parser.add_option( - '--setuptools', - dest='setuptools', - action='store_true', - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.") - - parser.add_option( - '--distribute', - dest='distribute', - action='store_true', - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.") - - parser.add_option( - '--unzip-setuptools', - action='store_true', - help="DEPRECATED. Retained only for backward compatibility. This option has no effect.") - - if 'extend_parser' in globals(): - extend_parser(parser) - - options, args = parser.parse_args() - - global logger - - if 'adjust_options' in globals(): - adjust_options(options, args) - - verbosity = options.verbose - options.quiet - logger = Logger([(Logger.level_for_integer(2 - verbosity), sys.stdout)]) - - if options.python and not os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'): - env = os.environ.copy() - interpreter = resolve_interpreter(options.python) - if interpreter == sys.executable: - logger.warn('Already using interpreter %s' % interpreter) - else: - logger.notify('Running virtualenv with interpreter %s' % interpreter) - env['VIRTUALENV_INTERPRETER_RUNNING'] = 'true' - file = __file__ - if file.endswith('.pyc'): - file = file[:-1] - popen = subprocess.Popen([interpreter, file] + sys.argv[1:], env=env) - raise SystemExit(popen.wait()) - - if not args: - print('You must provide a DEST_DIR') - parser.print_help() - sys.exit(2) - if len(args) > 1: - print('There must be only one argument: DEST_DIR (you gave %s)' % ( - ' '.join(args))) - parser.print_help() - sys.exit(2) - - home_dir = args[0] - - if os.path.exists(home_dir) and os.path.isfile(home_dir): - logger.fatal('ERROR: File already exists and is not a directory.') - logger.fatal('Please provide a different path or delete the file.') - sys.exit(3) - - if os.environ.get('WORKING_ENV'): - logger.fatal('ERROR: you cannot run virtualenv while in a workingenv') - logger.fatal('Please deactivate your workingenv, then re-run this script') - sys.exit(3) - - if 'PYTHONHOME' in os.environ: - logger.warn('PYTHONHOME is set. You *must* activate the virtualenv before using it') - del os.environ['PYTHONHOME'] - - if options.relocatable: - make_environment_relocatable(home_dir) - return - - create_environment(home_dir, - site_packages=options.system_site_packages, - clear=options.clear, - prompt=options.prompt, - search_dirs=options.search_dirs, - download=options.download, - no_setuptools=options.no_setuptools, - no_pip=options.no_pip, - no_wheel=options.no_wheel, - symlink=options.symlink) - if 'after_install' in globals(): - after_install(options, home_dir) - -def call_subprocess(cmd, show_stdout=True, - filter_stdout=None, cwd=None, - raise_on_returncode=True, extra_env=None, - remove_from_env=None, stdin=None): - cmd_parts = [] - for part in cmd: - if len(part) > 45: - part = part[:20]+"..."+part[-20:] - if ' ' in part or '\n' in part or '"' in part or "'" in part: - part = '"%s"' % part.replace('"', '\\"') - if hasattr(part, 'decode'): - try: - part = part.decode(sys.getdefaultencoding()) - except UnicodeDecodeError: - part = part.decode(sys.getfilesystemencoding()) - cmd_parts.append(part) - cmd_desc = ' '.join(cmd_parts) - if show_stdout: - stdout = None - else: - stdout = subprocess.PIPE - logger.debug("Running command %s" % cmd_desc) - if extra_env or remove_from_env: - env = os.environ.copy() - if extra_env: - env.update(extra_env) - if remove_from_env: - for varname in remove_from_env: - env.pop(varname, None) - else: - env = None - try: - proc = subprocess.Popen( - cmd, stderr=subprocess.STDOUT, - stdin=None if stdin is None else subprocess.PIPE, - stdout=stdout, - cwd=cwd, env=env) - except Exception: - e = sys.exc_info()[1] - logger.fatal( - "Error %s while executing command %s" % (e, cmd_desc)) - raise - all_output = [] - if stdout is not None: - if stdin is not None: - proc.stdin.write(stdin) - proc.stdin.close() - - stdout = proc.stdout - encoding = sys.getdefaultencoding() - fs_encoding = sys.getfilesystemencoding() - while 1: - line = stdout.readline() - try: - line = line.decode(encoding) - except UnicodeDecodeError: - line = line.decode(fs_encoding) - if not line: - break - line = line.rstrip() - all_output.append(line) - if filter_stdout: - level = filter_stdout(line) - if isinstance(level, tuple): - level, line = level - logger.log(level, line) - if not logger.stdout_level_matches(level): - logger.show_progress() - else: - logger.info(line) - else: - proc.communicate(stdin) - proc.wait() - if proc.returncode: - if raise_on_returncode: - if all_output: - logger.notify('Complete output from command %s:' % cmd_desc) - logger.notify('\n'.join(all_output) + '\n----------------------------------------') - raise OSError( - "Command %s failed with error code %s" - % (cmd_desc, proc.returncode)) - else: - logger.warn( - "Command %s had error code %s" - % (cmd_desc, proc.returncode)) - -def filter_install_output(line): - if line.strip().startswith('running'): - return Logger.INFO - return Logger.DEBUG - -def find_wheels(projects, search_dirs): - """Find wheels from which we can import PROJECTS. - - Scan through SEARCH_DIRS for a wheel for each PROJECT in turn. Return - a list of the first wheel found for each PROJECT - """ - - wheels = [] - - # Look through SEARCH_DIRS for the first suitable wheel. Don't bother - # about version checking here, as this is simply to get something we can - # then use to install the correct version. - for project in projects: - for dirname in search_dirs: - # This relies on only having "universal" wheels available. - # The pattern could be tightened to require -py2.py3-none-any.whl. - files = glob.glob(os.path.join(dirname, project + '-*.whl')) - if files: - wheels.append(os.path.abspath(files[0])) - break - else: - # We're out of luck, so quit with a suitable error - logger.fatal('Cannot find a wheel for %s' % (project,)) - - return wheels - -def install_wheel(project_names, py_executable, search_dirs=None, - download=False): - if search_dirs is None: - search_dirs = file_search_dirs() - - wheels = find_wheels(['setuptools', 'pip'], search_dirs) - pythonpath = os.pathsep.join(wheels) - - # PIP_FIND_LINKS uses space as the path separator and thus cannot have paths - # with spaces in them. Convert any of those to local file:// URL form. - try: - from urlparse import urljoin - from urllib import pathname2url - except ImportError: - from urllib.parse import urljoin - from urllib.request import pathname2url - def space_path2url(p): - if ' ' not in p: - return p - return urljoin('file:', pathname2url(os.path.abspath(p))) - findlinks = ' '.join(space_path2url(d) for d in search_dirs) - - SCRIPT = textwrap.dedent(""" - import sys - import pkgutil - import tempfile - import os - - try: - from pip._internal import main as _main - cert_data = pkgutil.get_data("pip._vendor.certifi", "cacert.pem") - except ImportError: - from pip import main as _main - cert_data = pkgutil.get_data("pip._vendor.requests", "cacert.pem") - - if cert_data is not None: - cert_file = tempfile.NamedTemporaryFile(delete=False) - cert_file.write(cert_data) - cert_file.close() - else: - cert_file = None - - try: - args = ["install", "--ignore-installed"] - if cert_file is not None: - args += ["--cert", cert_file.name] - args += sys.argv[1:] - - sys.exit(_main(args)) - finally: - if cert_file is not None: - os.remove(cert_file.name) - """).encode("utf8") - - cmd = [py_executable, '-'] + project_names - logger.start_progress('Installing %s...' % (', '.join(project_names))) - logger.indent += 2 - - env = { - "PYTHONPATH": pythonpath, - "JYTHONPATH": pythonpath, # for Jython < 3.x - "PIP_FIND_LINKS": findlinks, - "PIP_USE_WHEEL": "1", - "PIP_ONLY_BINARY": ":all:", - "PIP_USER": "0", - } - - if not download: - env["PIP_NO_INDEX"] = "1" - - try: - call_subprocess(cmd, show_stdout=False, extra_env=env, stdin=SCRIPT) - finally: - logger.indent -= 2 - logger.end_progress() - - -def create_environment(home_dir, site_packages=False, clear=False, - prompt=None, search_dirs=None, download=False, - no_setuptools=False, no_pip=False, no_wheel=False, - symlink=True): - """ - Creates a new environment in ``home_dir``. - - If ``site_packages`` is true, then the global ``site-packages/`` - directory will be on the path. - - If ``clear`` is true (default False) then the environment will - first be cleared. - """ - home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) - - py_executable = os.path.abspath(install_python( - home_dir, lib_dir, inc_dir, bin_dir, - site_packages=site_packages, clear=clear, symlink=symlink)) - - install_distutils(home_dir) - - to_install = [] - - if not no_setuptools: - to_install.append('setuptools') - - if not no_pip: - to_install.append('pip') - - if not no_wheel: - to_install.append('wheel') - - if to_install: - install_wheel( - to_install, - py_executable, - search_dirs, - download=download, - ) - - install_activate(home_dir, bin_dir, prompt) - - install_python_config(home_dir, bin_dir, prompt) - -def is_executable_file(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - -def path_locations(home_dir): - """Return the path locations for the environment (where libraries are, - where scripts go, etc)""" - home_dir = os.path.abspath(home_dir) - # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its - # prefix arg is broken: http://bugs.python.org/issue3386 - if is_win: - # Windows has lots of problems with executables with spaces in - # the name; this function will remove them (using the ~1 - # format): - mkdir(home_dir) - if ' ' in home_dir: - import ctypes - GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW - size = max(len(home_dir)+1, 256) - buf = ctypes.create_unicode_buffer(size) - try: - u = unicode - except NameError: - u = str - ret = GetShortPathName(u(home_dir), buf, size) - if not ret: - print('Error: the path "%s" has a space in it' % home_dir) - print('We could not determine the short pathname for it.') - print('Exiting.') - sys.exit(3) - home_dir = str(buf.value) - lib_dir = join(home_dir, 'Lib') - inc_dir = join(home_dir, 'Include') - bin_dir = join(home_dir, 'Scripts') - if is_jython: - lib_dir = join(home_dir, 'Lib') - inc_dir = join(home_dir, 'Include') - bin_dir = join(home_dir, 'bin') - elif is_pypy: - lib_dir = home_dir - inc_dir = join(home_dir, 'include') - bin_dir = join(home_dir, 'bin') - elif not is_win: - lib_dir = join(home_dir, 'lib', py_version) - inc_dir = join(home_dir, 'include', py_version + abiflags) - bin_dir = join(home_dir, 'bin') - return home_dir, lib_dir, inc_dir, bin_dir - - -def change_prefix(filename, dst_prefix): - prefixes = [sys.prefix] - - if is_darwin: - prefixes.extend(( - os.path.join("/Library/Python", sys.version[:3], "site-packages"), - os.path.join(sys.prefix, "Extras", "lib", "python"), - os.path.join("~", "Library", "Python", sys.version[:3], "site-packages"), - # Python 2.6 no-frameworks - os.path.join("~", ".local", "lib","python", sys.version[:3], "site-packages"), - # System Python 2.7 on OSX Mountain Lion - os.path.join("~", "Library", "Python", sys.version[:3], "lib", "python", "site-packages"))) - - if hasattr(sys, 'real_prefix'): - prefixes.append(sys.real_prefix) - if hasattr(sys, 'base_prefix'): - prefixes.append(sys.base_prefix) - prefixes = list(map(os.path.expanduser, prefixes)) - prefixes = list(map(os.path.abspath, prefixes)) - # Check longer prefixes first so we don't split in the middle of a filename - prefixes = sorted(prefixes, key=len, reverse=True) - filename = os.path.abspath(filename) - # On Windows, make sure drive letter is uppercase - if is_win and filename[0] in 'abcdefghijklmnopqrstuvwxyz': - filename = filename[0].upper() + filename[1:] - for i, prefix in enumerate(prefixes): - if is_win and prefix[0] in 'abcdefghijklmnopqrstuvwxyz': - prefixes[i] = prefix[0].upper() + prefix[1:] - for src_prefix in prefixes: - if filename.startswith(src_prefix): - _, relpath = filename.split(src_prefix, 1) - if src_prefix != os.sep: # sys.prefix == "/" - assert relpath[0] == os.sep - relpath = relpath[1:] - return join(dst_prefix, relpath) - assert False, "Filename %s does not start with any of these prefixes: %s" % \ - (filename, prefixes) - -def copy_required_modules(dst_prefix, symlink): - import imp - - for modname in REQUIRED_MODULES: - if modname in sys.builtin_module_names: - logger.info("Ignoring built-in bootstrap module: %s" % modname) - continue - try: - f, filename, _ = imp.find_module(modname) - except ImportError: - logger.info("Cannot import bootstrap module: %s" % modname) - else: - if f is not None: - f.close() - # special-case custom readline.so on OS X, but not for pypy: - if modname == 'readline' and sys.platform == 'darwin' and not ( - is_pypy or filename.endswith(join('lib-dynload', 'readline.so'))): - dst_filename = join(dst_prefix, 'lib', 'python%s' % sys.version[:3], 'readline.so') - elif modname == 'readline' and sys.platform == 'win32': - # special-case for Windows, where readline is not a - # standard module, though it may have been installed in - # site-packages by a third-party package - pass - else: - dst_filename = change_prefix(filename, dst_prefix) - copyfile(filename, dst_filename, symlink) - if filename.endswith('.pyc'): - pyfile = filename[:-1] - if os.path.exists(pyfile): - copyfile(pyfile, dst_filename[:-1], symlink) - -def copy_tcltk(src, dest, symlink): - """ copy tcl/tk libraries on Windows (issue #93) """ - for libversion in '8.5', '8.6': - for libname in 'tcl', 'tk': - srcdir = join(src, 'tcl', libname + libversion) - destdir = join(dest, 'tcl', libname + libversion) - # Only copy the dirs from the above combinations that exist - if os.path.exists(srcdir) and not os.path.exists(destdir): - copyfileordir(srcdir, destdir, symlink) - - -def subst_path(prefix_path, prefix, home_dir): - prefix_path = os.path.normpath(prefix_path) - prefix = os.path.normpath(prefix) - home_dir = os.path.normpath(home_dir) - if not prefix_path.startswith(prefix): - logger.warn('Path not in prefix %r %r', prefix_path, prefix) - return - return prefix_path.replace(prefix, home_dir, 1) - - -def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear, symlink=True): - """Install just the base environment, no distutils patches etc""" - if sys.executable.startswith(bin_dir): - print('Please use the *system* python to run this script') - return - - if clear: - rmtree(lib_dir) - ## FIXME: why not delete it? - ## Maybe it should delete everything with #!/path/to/venv/python in it - logger.notify('Not deleting %s', bin_dir) - - if hasattr(sys, 'real_prefix'): - logger.notify('Using real prefix %r' % sys.real_prefix) - prefix = sys.real_prefix - elif hasattr(sys, 'base_prefix'): - logger.notify('Using base prefix %r' % sys.base_prefix) - prefix = sys.base_prefix - else: - prefix = sys.prefix - mkdir(lib_dir) - fix_lib64(lib_dir, symlink) - stdlib_dirs = [os.path.dirname(os.__file__)] - if is_win: - stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), 'DLLs')) - elif is_darwin: - stdlib_dirs.append(join(stdlib_dirs[0], 'site-packages')) - if hasattr(os, 'symlink'): - logger.info('Symlinking Python bootstrap modules') - else: - logger.info('Copying Python bootstrap modules') - logger.indent += 2 - try: - # copy required files... - for stdlib_dir in stdlib_dirs: - if not os.path.isdir(stdlib_dir): - continue - for fn in os.listdir(stdlib_dir): - bn = os.path.splitext(fn)[0] - if fn != 'site-packages' and bn in REQUIRED_FILES: - copyfile(join(stdlib_dir, fn), join(lib_dir, fn), symlink) - # ...and modules - copy_required_modules(home_dir, symlink) - finally: - logger.indent -= 2 - # ...copy tcl/tk - if is_win: - copy_tcltk(prefix, home_dir, symlink) - mkdir(join(lib_dir, 'site-packages')) - import site - site_filename = site.__file__ - if site_filename.endswith('.pyc') or site_filename.endswith('.pyo'): - site_filename = site_filename[:-1] - elif site_filename.endswith('$py.class'): - site_filename = site_filename.replace('$py.class', '.py') - site_filename_dst = change_prefix(site_filename, home_dir) - site_dir = os.path.dirname(site_filename_dst) - writefile(site_filename_dst, SITE_PY) - writefile(join(site_dir, 'orig-prefix.txt'), prefix) - site_packages_filename = join(site_dir, 'no-global-site-packages.txt') - if not site_packages: - writefile(site_packages_filename, '') - - if is_pypy or is_win: - stdinc_dir = join(prefix, 'include') - else: - stdinc_dir = join(prefix, 'include', py_version + abiflags) - if os.path.exists(stdinc_dir): - copyfile(stdinc_dir, inc_dir, symlink) - else: - logger.debug('No include dir %s' % stdinc_dir) - - platinc_dir = distutils.sysconfig.get_python_inc(plat_specific=1) - if platinc_dir != stdinc_dir: - platinc_dest = distutils.sysconfig.get_python_inc( - plat_specific=1, prefix=home_dir) - if platinc_dir == platinc_dest: - # Do platinc_dest manually due to a CPython bug; - # not http://bugs.python.org/issue3386 but a close cousin - platinc_dest = subst_path(platinc_dir, prefix, home_dir) - if platinc_dest: - # PyPy's stdinc_dir and prefix are relative to the original binary - # (traversing virtualenvs), whereas the platinc_dir is relative to - # the inner virtualenv and ignores the prefix argument. - # This seems more evolved than designed. - copyfile(platinc_dir, platinc_dest, symlink) - - # pypy never uses exec_prefix, just ignore it - if sys.exec_prefix != prefix and not is_pypy: - if is_win: - exec_dir = join(sys.exec_prefix, 'lib') - elif is_jython: - exec_dir = join(sys.exec_prefix, 'Lib') - else: - exec_dir = join(sys.exec_prefix, 'lib', py_version) - for fn in os.listdir(exec_dir): - copyfile(join(exec_dir, fn), join(lib_dir, fn), symlink) - - if is_jython: - # Jython has either jython-dev.jar and javalib/ dir, or just - # jython.jar - for name in 'jython-dev.jar', 'javalib', 'jython.jar': - src = join(prefix, name) - if os.path.exists(src): - copyfile(src, join(home_dir, name), symlink) - # XXX: registry should always exist after Jython 2.5rc1 - src = join(prefix, 'registry') - if os.path.exists(src): - copyfile(src, join(home_dir, 'registry'), symlink=False) - copyfile(join(prefix, 'cachedir'), join(home_dir, 'cachedir'), - symlink=False) - - mkdir(bin_dir) - py_executable = join(bin_dir, os.path.basename(sys.executable)) - if 'Python.framework' in prefix: - # OS X framework builds cause validation to break - # https://github.com/pypa/virtualenv/issues/322 - if os.environ.get('__PYVENV_LAUNCHER__'): - del os.environ["__PYVENV_LAUNCHER__"] - if re.search(r'/Python(?:-32|-64)*$', py_executable): - # The name of the python executable is not quite what - # we want, rename it. - py_executable = os.path.join( - os.path.dirname(py_executable), 'python') - - logger.notify('New %s executable in %s', expected_exe, py_executable) - pcbuild_dir = os.path.dirname(sys.executable) - pyd_pth = os.path.join(lib_dir, 'site-packages', 'virtualenv_builddir_pyd.pth') - if is_win and os.path.exists(os.path.join(pcbuild_dir, 'build.bat')): - logger.notify('Detected python running from build directory %s', pcbuild_dir) - logger.notify('Writing .pth file linking to build directory for *.pyd files') - writefile(pyd_pth, pcbuild_dir) - else: - pcbuild_dir = None - if os.path.exists(pyd_pth): - logger.info('Deleting %s (not Windows env or not build directory python)' % pyd_pth) - os.unlink(pyd_pth) - - if sys.executable != py_executable: - ## FIXME: could I just hard link? - executable = sys.executable - shutil.copyfile(executable, py_executable) - make_exe(py_executable) - if is_win or is_cygwin: - pythonw = os.path.join(os.path.dirname(sys.executable), 'pythonw.exe') - if os.path.exists(pythonw): - logger.info('Also created pythonw.exe') - shutil.copyfile(pythonw, os.path.join(os.path.dirname(py_executable), 'pythonw.exe')) - python_d = os.path.join(os.path.dirname(sys.executable), 'python_d.exe') - python_d_dest = os.path.join(os.path.dirname(py_executable), 'python_d.exe') - if os.path.exists(python_d): - logger.info('Also created python_d.exe') - shutil.copyfile(python_d, python_d_dest) - elif os.path.exists(python_d_dest): - logger.info('Removed python_d.exe as it is no longer at the source') - os.unlink(python_d_dest) - - # we need to copy the DLL to enforce that windows will load the correct one. - # may not exist if we are cygwin. - if is_pypy: - py_executable_dlls = [ - ( - 'libpypy-c.dll', - 'libpypy_d-c.dll', - ), - ] - else: - py_executable_dlls = [ - ( - 'python%s.dll' % (sys.version_info[0]), - 'python%s_d.dll' % (sys.version_info[0]) - ), - ( - 'python%s%s.dll' % (sys.version_info[0], sys.version_info[1]), - 'python%s%s_d.dll' % (sys.version_info[0], sys.version_info[1]) - ) - ] - - for py_executable_dll, py_executable_dll_d in py_executable_dlls: - pythondll = os.path.join(os.path.dirname(sys.executable), py_executable_dll) - pythondll_d = os.path.join(os.path.dirname(sys.executable), py_executable_dll_d) - pythondll_d_dest = os.path.join(os.path.dirname(py_executable), py_executable_dll_d) - if os.path.exists(pythondll): - logger.info('Also created %s' % py_executable_dll) - shutil.copyfile(pythondll, os.path.join(os.path.dirname(py_executable), py_executable_dll)) - if os.path.exists(pythondll_d): - logger.info('Also created %s' % py_executable_dll_d) - shutil.copyfile(pythondll_d, pythondll_d_dest) - elif os.path.exists(pythondll_d_dest): - logger.info('Removed %s as the source does not exist' % pythondll_d_dest) - os.unlink(pythondll_d_dest) - if is_pypy: - # make a symlink python --> pypy-c - python_executable = os.path.join(os.path.dirname(py_executable), 'python') - if sys.platform in ('win32', 'cygwin'): - python_executable += '.exe' - logger.info('Also created executable %s' % python_executable) - copyfile(py_executable, python_executable, symlink) - - if is_win: - for name in ['libexpat.dll', - 'libeay32.dll', 'ssleay32.dll', 'sqlite3.dll', - 'tcl85.dll', 'tk85.dll']: - src = join(prefix, name) - if os.path.exists(src): - copyfile(src, join(bin_dir, name), symlink) - - for d in sys.path: - if d.endswith('lib_pypy'): - break - else: - logger.fatal('Could not find lib_pypy in sys.path') - raise SystemExit(3) - logger.info('Copying lib_pypy') - copyfile(d, os.path.join(home_dir, 'lib_pypy'), symlink) - - if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe: - secondary_exe = os.path.join(os.path.dirname(py_executable), - expected_exe) - py_executable_ext = os.path.splitext(py_executable)[1] - if py_executable_ext.lower() == '.exe': - # python2.4 gives an extension of '.4' :P - secondary_exe += py_executable_ext - if os.path.exists(secondary_exe): - logger.warn('Not overwriting existing %s script %s (you must use %s)' - % (expected_exe, secondary_exe, py_executable)) - else: - logger.notify('Also creating executable in %s' % secondary_exe) - shutil.copyfile(sys.executable, secondary_exe) - make_exe(secondary_exe) - - if '.framework' in prefix: - if 'Python.framework' in prefix: - logger.debug('MacOSX Python framework detected') - # Make sure we use the embedded interpreter inside - # the framework, even if sys.executable points to - # the stub executable in ${sys.prefix}/bin - # See http://groups.google.com/group/python-virtualenv/ - # browse_thread/thread/17cab2f85da75951 - original_python = os.path.join( - prefix, 'Resources/Python.app/Contents/MacOS/Python') - if 'EPD' in prefix: - logger.debug('EPD framework detected') - original_python = os.path.join(prefix, 'bin/python') - shutil.copy(original_python, py_executable) - - # Copy the framework's dylib into the virtual - # environment - virtual_lib = os.path.join(home_dir, '.Python') - - if os.path.exists(virtual_lib): - os.unlink(virtual_lib) - copyfile( - os.path.join(prefix, 'Python'), - virtual_lib, - symlink) - - # And then change the install_name of the copied python executable - try: - mach_o_change(py_executable, - os.path.join(prefix, 'Python'), - '@executable_path/../.Python') - except: - e = sys.exc_info()[1] - logger.warn("Could not call mach_o_change: %s. " - "Trying to call install_name_tool instead." % e) - try: - call_subprocess( - ["install_name_tool", "-change", - os.path.join(prefix, 'Python'), - '@executable_path/../.Python', - py_executable]) - except: - logger.fatal("Could not call install_name_tool -- you must " - "have Apple's development tools installed") - raise - - if not is_win: - # Ensure that 'python', 'pythonX' and 'pythonX.Y' all exist - py_exe_version_major = 'python%s' % sys.version_info[0] - py_exe_version_major_minor = 'python%s.%s' % ( - sys.version_info[0], sys.version_info[1]) - py_exe_no_version = 'python' - required_symlinks = [ py_exe_no_version, py_exe_version_major, - py_exe_version_major_minor ] - - py_executable_base = os.path.basename(py_executable) - - if py_executable_base in required_symlinks: - # Don't try to symlink to yourself. - required_symlinks.remove(py_executable_base) - - for pth in required_symlinks: - full_pth = join(bin_dir, pth) - if os.path.exists(full_pth): - os.unlink(full_pth) - if symlink: - os.symlink(py_executable_base, full_pth) - else: - copyfile(py_executable, full_pth, symlink) - - cmd = [py_executable, '-c', 'import sys;out=sys.stdout;' - 'getattr(out, "buffer", out).write(sys.prefix.encode("utf-8"))'] - logger.info('Testing executable with %s %s "%s"' % tuple(cmd)) - try: - proc = subprocess.Popen(cmd, - stdout=subprocess.PIPE) - proc_stdout, proc_stderr = proc.communicate() - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.EACCES: - logger.fatal('ERROR: The executable %s could not be run: %s' % (py_executable, e)) - sys.exit(100) - else: - raise e - - proc_stdout = proc_stdout.strip().decode("utf-8") - proc_stdout = os.path.normcase(os.path.abspath(proc_stdout)) - norm_home_dir = os.path.normcase(os.path.abspath(home_dir)) - if hasattr(norm_home_dir, 'decode'): - norm_home_dir = norm_home_dir.decode(sys.getfilesystemencoding()) - if proc_stdout != norm_home_dir: - logger.fatal( - 'ERROR: The executable %s is not functioning' % py_executable) - logger.fatal( - 'ERROR: It thinks sys.prefix is %r (should be %r)' - % (proc_stdout, norm_home_dir)) - logger.fatal( - 'ERROR: virtualenv is not compatible with this system or executable') - if is_win: - logger.fatal( - 'Note: some Windows users have reported this error when they ' - 'installed Python for "Only this user" or have multiple ' - 'versions of Python installed. Copying the appropriate ' - 'PythonXX.dll to the virtualenv Scripts/ directory may fix ' - 'this problem.') - sys.exit(100) - else: - logger.info('Got sys.prefix result: %r' % proc_stdout) - - pydistutils = os.path.expanduser('~/.pydistutils.cfg') - if os.path.exists(pydistutils): - logger.notify('Please make sure you remove any previous custom paths from ' - 'your %s file.' % pydistutils) - ## FIXME: really this should be calculated earlier - - fix_local_scheme(home_dir, symlink) - - if site_packages: - if os.path.exists(site_packages_filename): - logger.info('Deleting %s' % site_packages_filename) - os.unlink(site_packages_filename) - - return py_executable - - -def install_activate(home_dir, bin_dir, prompt=None): - if is_win or is_jython and os._name == 'nt': - files = { - 'activate.bat': ACTIVATE_BAT, - 'deactivate.bat': DEACTIVATE_BAT, - 'activate.ps1': ACTIVATE_PS, - } - - # MSYS needs paths of the form /c/path/to/file - drive, tail = os.path.splitdrive(home_dir.replace(os.sep, '/')) - home_dir_msys = (drive and "/%s%s" or "%s%s") % (drive[:1], tail) - - # Run-time conditional enables (basic) Cygwin compatibility - home_dir_sh = ("""$(if [ "$OSTYPE" "==" "cygwin" ]; then cygpath -u '%s'; else echo '%s'; fi;)""" % - (home_dir, home_dir_msys)) - files['activate'] = ACTIVATE_SH.replace('__VIRTUAL_ENV__', home_dir_sh) - - else: - files = {'activate': ACTIVATE_SH} - - # suppling activate.fish in addition to, not instead of, the - # bash script support. - files['activate.fish'] = ACTIVATE_FISH - - # same for csh/tcsh support... - files['activate.csh'] = ACTIVATE_CSH - - files['activate_this.py'] = ACTIVATE_THIS - - install_files(home_dir, bin_dir, prompt, files) - -def install_files(home_dir, bin_dir, prompt, files): - if hasattr(home_dir, 'decode'): - home_dir = home_dir.decode(sys.getfilesystemencoding()) - vname = os.path.basename(home_dir) - for name, content in files.items(): - content = content.replace('__VIRTUAL_PROMPT__', prompt or '') - content = content.replace('__VIRTUAL_WINPROMPT__', prompt or '(%s)' % vname) - content = content.replace('__VIRTUAL_ENV__', home_dir) - content = content.replace('__VIRTUAL_NAME__', vname) - content = content.replace('__BIN_NAME__', os.path.basename(bin_dir)) - writefile(os.path.join(bin_dir, name), content) - -def install_python_config(home_dir, bin_dir, prompt=None): - if sys.platform == 'win32' or is_jython and os._name == 'nt': - files = {} - else: - files = {'python-config': PYTHON_CONFIG} - install_files(home_dir, bin_dir, prompt, files) - for name, content in files.items(): - make_exe(os.path.join(bin_dir, name)) - -def install_distutils(home_dir): - distutils_path = change_prefix(distutils.__path__[0], home_dir) - mkdir(distutils_path) - ## FIXME: maybe this prefix setting should only be put in place if - ## there's a local distutils.cfg with a prefix setting? - home_dir = os.path.abspath(home_dir) - ## FIXME: this is breaking things, removing for now: - #distutils_cfg = DISTUTILS_CFG + "\n[install]\nprefix=%s\n" % home_dir - writefile(os.path.join(distutils_path, '__init__.py'), DISTUTILS_INIT) - writefile(os.path.join(distutils_path, 'distutils.cfg'), DISTUTILS_CFG, overwrite=False) - -def fix_local_scheme(home_dir, symlink=True): - """ - Platforms that use the "posix_local" install scheme (like Ubuntu with - Python 2.7) need to be given an additional "local" location, sigh. - """ - try: - import sysconfig - except ImportError: - pass - else: - if sysconfig._get_default_scheme() == 'posix_local': - local_path = os.path.join(home_dir, 'local') - if not os.path.exists(local_path): - os.mkdir(local_path) - for subdir_name in os.listdir(home_dir): - if subdir_name == 'local': - continue - copyfile(os.path.abspath(os.path.join(home_dir, subdir_name)), \ - os.path.join(local_path, subdir_name), symlink) - -def fix_lib64(lib_dir, symlink=True): - """ - Some platforms (particularly Gentoo on x64) put things in lib64/pythonX.Y - instead of lib/pythonX.Y. If this is such a platform we'll just create a - symlink so lib64 points to lib - """ - # PyPy's library path scheme is not affected by this. - # Return early or we will die on the following assert. - if is_pypy: - logger.debug('PyPy detected, skipping lib64 symlinking') - return - # Check we have a lib64 library path - if not [p for p in distutils.sysconfig.get_config_vars().values() - if isinstance(p, basestring) and 'lib64' in p]: - return - - logger.debug('This system uses lib64; symlinking lib64 to lib') - - assert os.path.basename(lib_dir) == 'python%s' % sys.version[:3], ( - "Unexpected python lib dir: %r" % lib_dir) - lib_parent = os.path.dirname(lib_dir) - top_level = os.path.dirname(lib_parent) - lib_dir = os.path.join(top_level, 'lib') - lib64_link = os.path.join(top_level, 'lib64') - assert os.path.basename(lib_parent) == 'lib', ( - "Unexpected parent dir: %r" % lib_parent) - if os.path.lexists(lib64_link): - return - if symlink: - os.symlink('lib', lib64_link) - else: - copyfile('lib', lib64_link) - -def resolve_interpreter(exe): - """ - If the executable given isn't an absolute path, search $PATH for the interpreter - """ - # If the "executable" is a version number, get the installed executable for - # that version - orig_exe = exe - python_versions = get_installed_pythons() - if exe in python_versions: - exe = python_versions[exe] - - if os.path.abspath(exe) != exe: - exe = distutils.spawn.find_executable(exe) or exe - if not os.path.exists(exe): - logger.fatal('The path %s (from --python=%s) does not exist' % (exe, orig_exe)) - raise SystemExit(3) - if not is_executable(exe): - logger.fatal('The path %s (from --python=%s) is not an executable file' % (exe, orig_exe)) - raise SystemExit(3) - return exe - -def is_executable(exe): - """Checks a file is executable""" - return os.path.isfile(exe) and os.access(exe, os.X_OK) - -############################################################ -## Relocating the environment: - -def make_environment_relocatable(home_dir): - """ - Makes the already-existing environment use relative paths, and takes out - the #!-based environment selection in scripts. - """ - home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir) - activate_this = os.path.join(bin_dir, 'activate_this.py') - if not os.path.exists(activate_this): - logger.fatal( - 'The environment doesn\'t have a file %s -- please re-run virtualenv ' - 'on this environment to update it' % activate_this) - fixup_scripts(home_dir, bin_dir) - fixup_pth_and_egg_link(home_dir) - ## FIXME: need to fix up distutils.cfg - -OK_ABS_SCRIPTS = ['python', 'python%s' % sys.version[:3], - 'activate', 'activate.bat', 'activate_this.py', - 'activate.fish', 'activate.csh'] - -def fixup_scripts(home_dir, bin_dir): - if is_win: - new_shebang_args = ( - '%s /c' % os.path.normcase(os.environ.get('COMSPEC', 'cmd.exe')), - '', '.exe') - else: - new_shebang_args = ('/usr/bin/env', sys.version[:3], '') - - # This is what we expect at the top of scripts: - shebang = '#!%s' % os.path.normcase(os.path.join( - os.path.abspath(bin_dir), 'python%s' % new_shebang_args[2])) - # This is what we'll put: - new_shebang = '#!%s python%s%s' % new_shebang_args - - for filename in os.listdir(bin_dir): - filename = os.path.join(bin_dir, filename) - if not os.path.isfile(filename): - # ignore subdirs, e.g. .svn ones. - continue - lines = None - with open(filename, 'rb') as f: - try: - lines = f.read().decode('utf-8').splitlines() - except UnicodeDecodeError: - # This is probably a binary program instead - # of a script, so just ignore it. - continue - if not lines: - logger.warn('Script %s is an empty file' % filename) - continue - - old_shebang = lines[0].strip() - old_shebang = old_shebang[0:2] + os.path.normcase(old_shebang[2:]) - - if not old_shebang.startswith(shebang): - if os.path.basename(filename) in OK_ABS_SCRIPTS: - logger.debug('Cannot make script %s relative' % filename) - elif lines[0].strip() == new_shebang: - logger.info('Script %s has already been made relative' % filename) - else: - logger.warn('Script %s cannot be made relative (it\'s not a normal script that starts with %s)' - % (filename, shebang)) - continue - logger.notify('Making script %s relative' % filename) - script = relative_script([new_shebang] + lines[1:]) - with open(filename, 'wb') as f: - f.write('\n'.join(script).encode('utf-8')) - - -def relative_script(lines): - "Return a script that'll work in a relocatable environment." - activate = "import os; activate_this=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'activate_this.py'); exec(compile(open(activate_this).read(), activate_this, 'exec'), dict(__file__=activate_this)); del os, activate_this" - # Find the last future statement in the script. If we insert the activation - # line before a future statement, Python will raise a SyntaxError. - activate_at = None - for idx, line in reversed(list(enumerate(lines))): - if line.split()[:3] == ['from', '__future__', 'import']: - activate_at = idx + 1 - break - if activate_at is None: - # Activate after the shebang. - activate_at = 1 - return lines[:activate_at] + ['', activate, ''] + lines[activate_at:] - -def fixup_pth_and_egg_link(home_dir, sys_path=None): - """Makes .pth and .egg-link files use relative paths""" - home_dir = os.path.normcase(os.path.abspath(home_dir)) - if sys_path is None: - sys_path = sys.path - for path in sys_path: - if not path: - path = '.' - if not os.path.isdir(path): - continue - path = os.path.normcase(os.path.abspath(path)) - if not path.startswith(home_dir): - logger.debug('Skipping system (non-environment) directory %s' % path) - continue - for filename in os.listdir(path): - filename = os.path.join(path, filename) - if filename.endswith('.pth'): - if not os.access(filename, os.W_OK): - logger.warn('Cannot write .pth file %s, skipping' % filename) - else: - fixup_pth_file(filename) - if filename.endswith('.egg-link'): - if not os.access(filename, os.W_OK): - logger.warn('Cannot write .egg-link file %s, skipping' % filename) - else: - fixup_egg_link(filename) - -def fixup_pth_file(filename): - lines = [] - prev_lines = [] - with open(filename) as f: - prev_lines = f.readlines() - for line in prev_lines: - line = line.strip() - if (not line or line.startswith('#') or line.startswith('import ') - or os.path.abspath(line) != line): - lines.append(line) - else: - new_value = make_relative_path(filename, line) - if line != new_value: - logger.debug('Rewriting path %s as %s (in %s)' % (line, new_value, filename)) - lines.append(new_value) - if lines == prev_lines: - logger.info('No changes to .pth file %s' % filename) - return - logger.notify('Making paths in .pth file %s relative' % filename) - with open(filename, 'w') as f: - f.write('\n'.join(lines) + '\n') - -def fixup_egg_link(filename): - with open(filename) as f: - link = f.readline().strip() - if os.path.abspath(link) != link: - logger.debug('Link in %s already relative' % filename) - return - new_link = make_relative_path(filename, link) - logger.notify('Rewriting link %s in %s as %s' % (link, filename, new_link)) - with open(filename, 'w') as f: - f.write(new_link) - -def make_relative_path(source, dest, dest_is_directory=True): - """ - Make a filename relative, where the filename is dest, and it is - being referred to from the filename source. - - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/usr/share/another-place/src/Directory') - '../another-place/src/Directory' - >>> make_relative_path('/usr/share/something/a-file.pth', - ... '/home/user/src/Directory') - '../../../home/user/src/Directory' - >>> make_relative_path('/usr/share/a-file.pth', '/usr/share/') - './' - """ - source = os.path.dirname(source) - if not dest_is_directory: - dest_filename = os.path.basename(dest) - dest = os.path.dirname(dest) - dest = os.path.normpath(os.path.abspath(dest)) - source = os.path.normpath(os.path.abspath(source)) - dest_parts = dest.strip(os.path.sep).split(os.path.sep) - source_parts = source.strip(os.path.sep).split(os.path.sep) - while dest_parts and source_parts and dest_parts[0] == source_parts[0]: - dest_parts.pop(0) - source_parts.pop(0) - full_parts = ['..']*len(source_parts) + dest_parts - if not dest_is_directory: - full_parts.append(dest_filename) - if not full_parts: - # Special case for the current directory (otherwise it'd be '') - return './' - return os.path.sep.join(full_parts) - - - -############################################################ -## Bootstrap script creation: - -def create_bootstrap_script(extra_text, python_version=''): - """ - Creates a bootstrap script, which is like this script but with - extend_parser, adjust_options, and after_install hooks. - - This returns a string that (written to disk of course) can be used - as a bootstrap script with your own customizations. The script - will be the standard virtualenv.py script, with your extra text - added (your extra text should be Python code). - - If you include these functions, they will be called: - - ``extend_parser(optparse_parser)``: - You can add or remove options from the parser here. - - ``adjust_options(options, args)``: - You can change options here, or change the args (if you accept - different kinds of arguments, be sure you modify ``args`` so it is - only ``[DEST_DIR]``). - - ``after_install(options, home_dir)``: - - After everything is installed, this function is called. This - is probably the function you are most likely to use. An - example would be:: - - def after_install(options, home_dir): - subprocess.call([join(home_dir, 'bin', 'easy_install'), - 'MyPackage']) - subprocess.call([join(home_dir, 'bin', 'my-package-script'), - 'setup', home_dir]) - - This example immediately installs a package, and runs a setup - script from that package. - - If you provide something like ``python_version='2.5'`` then the - script will start with ``#!/usr/bin/env python2.5`` instead of - ``#!/usr/bin/env python``. You can use this when the script must - be run with a particular Python version. - """ - filename = __file__ - if filename.endswith('.pyc'): - filename = filename[:-1] - with codecs.open(filename, 'r', encoding='utf-8') as f: - content = f.read() - py_exe = 'python%s' % python_version - content = (('#!/usr/bin/env %s\n' % py_exe) - + '## WARNING: This file is generated\n' - + content) - return content.replace('##EXT' 'END##', extra_text) - -##EXTEND## - -def convert(s): - b = base64.b64decode(s.encode('ascii')) - return zlib.decompress(b).decode('utf-8') - -##file site.py -SITE_PY = convert(""" -eJzFPf1z2zaWv/OvwMqToZTKdOJ0e3tO3RsncVrfuYm3yc7m1vXoKAmyWFMkS5C2tTd3f/u9DwAE -+CHb2+6cphNLJPDw8PC+8PAeOhqNTopCZkuxyZd1KoWScblYiyKu1kqs8lJU66Rc7hdxWW3h6eIm -vpZKVLlQWxVhqygInv/GT/BcfF4nyqAA3+K6yjdxlSziNN2KZFPkZSWXYlmXSXYtkiypkjhN/g4t -8iwSz387BsFZJmDmaSJLcStLBXCVyFfiYlut80yM6wLn/DL6Y/xqMhVqUSZFBQ1KjTNQZB1XQSbl -EtCElrUCUiaV3FeFXCSrZGEb3uV1uhRFGi+k+K//4qlR0zAMVL6Rd2tZSpEBMgBTAqwC8YCvSSkW -+VJGQryRixgH4OcNsQKGNsU1U0jGLBdpnl3DnDK5kErF5VaM53VFgAhlscwBpwQwqJI0De7y8kZN -YElpPe7gkYiZPfzJMHvAPHH8LucAjh+z4C9Zcj9l2MA9CK5aM9uUcpXcixjBwk95Lxcz/WycrMQy -Wa2ABlk1wSYBI6BEmswPClqOb/UKfXdAWFmujGEMiShzY35JPaLgrBJxqoBt6wJppAjzd3KexBlQ -I7uF4QAikDToG2eZqMqOQ7MTOQAocR0rkJKNEuNNnGTArD/GC0L7r0m2zO/UhCgAq6XEL7Wq3PmP -ewgArR0CTANcLLOadZYmNzLdTgCBz4B9KVWdVigQy6SUiyovE6kIAKC2FfIekJ6KuJSahMyZRm6n -RH+iSZLhwqKAocDjSyTJKrmuS5IwsUqAc4Er3n/8Sbw7fXN28kHzmAHGMnu9AZwBCi20gxMMIA5q -VR6kOQh0FJzjHxEvlyhk1zg+4NU0OHhwpYMxzL2I2n2cBQey68XVw8AcK1AmNFZA/f4bukzVGujz -Pw+sdxCcDFGFJs7f7tY5yGQWb6RYx8xfyBnBtxrOd1FRrV8DNyiEUwGpFC4OIpggPCCJS7NxnklR -AIulSSYnAVBoTm39VQRW+JBn+7TWLU4ACGWQwUvn2YRGzCRMtAvrNeoL03hLM9NNArvOm7wkxQH8 -ny1IF6VxdkM4KmIo/jaX10mWIULIC0G4F9LA6iYBTlxG4pxakV4wjUTI2otbokjUwEvIdMCT8j7e -FKmcsviibt2tRmgwWQmz1ilzHLSsSL3SqjVT7eW9w+hLi+sIzWpdSgBezz2hW+X5VMxBZxM2Rbxh -8arucuKcoEeeqBPyBLWEvvgdKHqiVL2R9iXyCmgWYqhgladpfgckOwoCIfawkTHKPnPCW3gH/wJc -/DeV1WIdBM5IFrAGhcgPgUIgYBJkprlaI+Fxm2bltpJJMtYUebmUJQ31OGIfMOKPbIxzDT7klTZq -PF1c5XyTVKiS5tpkJmzxsrBi/fia5w3TAMutiGamaUOnDU4vLdbxXBqXZC5XKAl6kV7bZYcxg54x -yRZXYsNWBt4BWWTCFqRfsaDSWVWSnACAwcIXZ0lRp9RIIYOJGAbaFAR/E6NJz7WzBOzNZjlAhcTm -ewH2B3D7O4jR3ToB+iwAAmgY1FKwfPOkKtFBaPRR4Bt905/HB049W2nbxEOu4iTVVj7OgjN6eFqW -JL4LWWCvqSaGghlmFbp21xnQEcV8NBoFgXGHtsp8zVVQldsjYAVhxpnN5nWChm82Q1Ovf6iARxHO -wF43287CAw1hOn0AKjldVmW+wdd2bp9AmcBY2CPYExekZSQ7yB4nvkbyuSq9ME3RdjvsLFAPBRc/ -nb4/+3L6SRyLy0alTdv67ArGPM1iYGuyCMBUrWEbXQYtUfElqPvEezDvxBRgz6g3ia+Mqxp4F1D/ -XNb0Gqax8F4Gpx9O3pyfzv7y6fSn2aezz6eAINgZGezRlNE81uAwqgiEA7hyqSJtX4NOD3rw5uST -fRDMEjX75mtgN3gyvpYVMHE5hhlPRbiJ7xUwaDilphPEsdMALHg4mYjvxOHz568OCVqxLbYADMyu -0xQfzrRFnyXZKg8n1PgXdumPWUlp/+3y6OsrcXwswl/i2zgMwIdqmjJL/Eji9HlbSOhawZ9xriZB -sJQrEL0biQI6fk5+8YQ7wJJAy1zb6V/yJDPvmSvdIUh/jKkH4DCbLdJYKWw8m4VABOrQ84EOETvX -KHVj6Fhs3a4TjQp+SgkLm2GXKf7Tg2I8p36IBqPodjGNQFw3i1hJbkXTh36zGeqs2WysBwRhJokB -h4vVUChME9RZZQJ+LXEe6rC5ylP8ifBRC5AA4tYKtSQukt46RbdxWks1diYFRByPW2RERZso4kdw -UcZgiZulm0za1DQ8A82AfGkOWrRsUQ4/e+DvgLoymzjc6PHei2mGmP477zQIB3A5Q1T3SrWgsHYU -F6cX4tWLw310Z2DPubTU8ZqjhU6yWtqHK1gtIw+MMPcy8uLSZYV6Fp8e7Ya5iezKdFlhpZe4lJv8 -Vi4BW2RgZ5XFT/QGduYwj0UMqwh6nfwBVqHGb4xxH8qzB2lB3wGotyEoZv3N0u9xMEBmChQRb6yJ -1HrXz6awKPPbBJ2N+Va/BFsJyhItpnFsAmfhPCZDkwgaArzgDCl1J0NQh2XNDivhjSDRXiwbxRoR -uHPU1Ff09SbL77IZ74SPUemOJ5Z1UbA082KDZgn2xHuwQoBkDhu7hmgMBVx+gbK1D8jD9GG6QFna -WwAgMPSKtmsOLLPVoynyrhGHRRiT14KEt5ToL9yaIWirZYjhQKK3kX1gtARCgslZBWdVg2YylDXT -DAZ2SOJz3XnEW1AfQIuKEZjNsYbGjQz9Lo9AOYtzVyk5/dAif/nyhdlGrSm+gojNcdLoQqzIWEbF -FgxrAjrBeGQcrSE2uAPnFsDUSrOm2P8k8oK9MVjPCy3b4AfA7q6qiqODg7u7u0hHF/Ly+kCtDv74 -p2+++dML1onLJfEPTMeRFh1qiw7oHXq00bfGAn1nVq7Fj0nmcyPBGkvyysgVRfy+r5NlLo72J1Z/ -Ihc3Zhr/Na4MKJCZGZSpDLQdNRg9U/vPoldqJJ6RdbZtxxP2S7RJtVbMt7rQo8rBEwC/ZZHXaKob -TlDiK7BusENfynl9HdrBPRtpfsBUUU7Hlgf2X14hBj5nGL4ypniGWoLYAi2+Q/qfmG1i8o60hkDy -oonq7J63/VrMEHf5eHm3vqYjNGaGiULuQInwmzxaAG3jruTgR7u2aPcc19Z8PENgLH1gmFc7lmMU -HMIF12LqSp3D1ejxgjTdsWoGBeOqRlDQ4CTOmdoaHNnIEEGid2M2+7ywugXQqRU5NPEBswrQwh2n -Y+3arOB4QsgDx+IlPZHgIh913r3gpa3TlAI6LR71qMKAvYVGO50DX44NgKkYlX8ZcUuzTfnYWhRe -gx5gOceAkMFWHWbCN64PONob9bBTx+oP9WYa94HARRpzLOpR0AnlYx6hVCBNxdjvOcTilrjdwXZa -HGIqs0wk0mpAuNrKo1eodhqmVZKh7nUWKVqkOXjFVisSIzXvfWeB9kH4uM+YaQnUZGjI4TQ6Jm/P -E8BQt8Pw2XWNgQY3DoMYbRJF1g3JtIZ/wK2g+AYFo4CWBM2CeayU+RP7HWTOzld/GWAPS2hkCLfp -kBvSsRgajnm/J5CMOhoDUpABCbvCSK4jq4MUOMxZIE+44bUclG6CESmQM8eCkJoB3Omlt8HBJxGe -gJCEIuT7SslCfCVGsHxtUX2c7v5dudQEIcZOA3IVdPTi2I1sOFGN41aUw2doP75BZyVFDhw8B5fH -DfS7bG6Y1gZdwFn3FbdFCjQyxWFGExfVK0MYN5j8h2OnRUMsM4hhKG8g70jHjDQJ7HJr0LDgBoy3 -5u2x9GM3YoF9x2GuDuXmHvZ/YZmoRa5Cipm0YxfuR3NFlzYW2/NkPoI/3gKMJlceJJnq+AVGWf6B -QUIPetgH3ZsshkWWcXmXZCEpME2/Y39pOnhYUnpG7uATbacOYKIY8Tx4X4KA0NHnAYgTagLYlctQ -abe/C3bnFEcWLncfeW7z5dGrqy5xp0MRHvvpX6rT+6qMFa5WyovGQoGr1TXgqHRhcnG21YeX+nAb -twllrmAXKT5++iKQEBzXvYu3T5t6w/CIzYNz8j4GddBrD5KrNTtiF0AEtSIyykH4dI58PLJPndyO -iT0ByJMYZseiGEiaT/4ROLsWCsbYX24zjKO1VQZ+4PU3X896IqMukt98PXpglBYx+sR+3PIE7cic -VLBrtqWMU3I1nD4UVMwa1rFtignrc9r+aR676vE5NVo29t3fAj8GCobUJfgIL6YN2bpTxY/vTg3C -03ZqB7DObtV89mgRYG+fz3+BHbLSQbXbOEnpXAEmv7+PytVs7jle0a89PEg7FYxDgr79l7p8AdwQ -cjRh0p2OdsZOTMC5ZxdsPkWsuqjs6RyC5gjMywtwjz+HFU6ve+B7Bge/r7p8IiBvTqMeMmpbbIZ4 -wQclhz1K9gnzfvqMf9dZP27mw4L1/zHLF/+cST5hKgaaNh4+rH5iuXbXAHuEeRpwO3e4hd2h+axy -ZZw7VklKPEfd9VzcUboCxVbxpAigLNnv64GDUqoPvd/WZclH16QCC1nu43HsVGCmlvH8ek3Mnjj4 -ICvExDZbUKzayevJ+4Qv1NFnO5Ow2Tf0c+c6NzErmd0mJfQFhTsOf/j442nYb0IwjgudHm9FHu83 -INwnMG6oiRM+pQ9T6Cld/nH10d66+AQ1GQEmIqzJ1iVsJxBs4gj9a/BARMg7sOVjdtyhL9ZycTOT -lDqAbIpdnaD4W3yNmNiMAj//S8UrSmKDmSzSGmnFjjdmH67qbEHnI5UE/0qnCmPqECUEcPhvlcbX -Ykydlxh60txI0anbuNTeZ1HmmJwq6mR5cJ0shfy1jlPc1svVCnDBwyv9KuLhKQIl3nFOAyctKrmo -y6TaAglileuzP0p/cBrOtzzRsYckH/MwATEh4kh8wmnjeybc0pDLBAf8Ew+cJO67sYOTrBDRc3if -5TMcdUY5vlNGqnsuT4+D9gg5ABgBUJj/aKIjd/4bSa/cA0Zac5eoqCU9UrqRhpycMYQynmCkg3/T -T58RXd4awPJ6GMvr3Vhet7G87sXy2sfyejeWrkjgwtqglZGEvsBV+1ijN9/GjTnxMKfxYs3tMPcT -czwBoijMBtvIFKdAe5EtPt8jIKS2nQNnetjkzyScVFrmHALXIJH78RBLb+ZN8rrTmbJxdGeeinFn -h3KI/L4HUUSpYnPqzvK2jKs48uTiOs3nILYW3WkDYCra6UQcK81uZ3OO7rYs1ejiPz//8PEDNkdQ -I5PeQN1wEdGw4FTGz+PyWnWlqdn8FcCO1NJPxKFuGuDeIyNrPMoe//OOMjyQccQdZSjkogAPgLK6 -bDM39ykMW891kpR+zkzOh03HYpRVo2ZSA0Q6ubh4d/L5ZEQhv9H/jlyBMbT1pcPFx7SwDbr+m9vc -Uhz7gFDr2FZj/Nw5ebRuOOJhG2vAdjzf1oPDxxjs3jCBP8t/KqVgSYBQkQ7+PoVQj945/Kb9UIc+ -hhE7yX/uyRo7K/adI3uOi+KIft+xQ3sA/7AT9xgzIIB2ocZmZ9DslVtK35rXHRR1gD7S1/vNe832 -1qu9k/EpaifR4wA6lLXNht0/75yGjZ6S1ZvT788+nJ+9uTj5/IPjAqIr9/HTwaE4/fGLoPwQNGDs -E8WYGlFhJhIYFrfQSSxz+K/GyM+yrjhIDL3enZ/rk5oNlrpg7jPanAiecxqThcZBM45C24c6/wgx -SvUGyakponQdqjnC/dKG61lUrvOjqVRpjs5qrbdeulbM1JTRuXYE0geNXVIwCE4xg1eUxV6ZXWHJ -J4C6zqoHKW2jbWJISkHBTrqAc/5lTle8QCl1hidNZ63oL0MX1/AqUkWawE7udWhlSXfD9JiGcfRD -e8DNePVpQKc7jKwb8qwHsUCr9Trkuen+k4bRfq0Bw4bB3sG8M0npIZSBjcltIsRGfJITynv4apde -r4GCBcODvgoX0TBdArOPYXMt1glsIIAn12B9cZ8AEFor4R8IHDnRAZljdkb4drPc/3OoCeK3/vnn -nuZVme7/TRSwCxKcShT2ENNt/A42PpGMxOnH95OQkaPUXPHnGssDwCGhAKgj7ZS/xCfos7GS6Urn -l/j6AF9oP4Fet7qXsih1937XOEQJeKbG5DU8U4Z+IaZ7WdhTnMqkBRorHyxmWEHopiGYz574tJZp -qvPdz96dn4LviMUYKEF87nYKw3G8BI/QdfIdVzi2QOEBO7wukY1LdGEpyWIZec16g9YoctTby8uw -60SB4W6vThS4jBPloj3GaTMsU04QISvDWphlZdZutUEKu22I4igzzBKzi5ISWH2eAF6mpzFviWCv -hKUeJgLPp8hJVpmMxTRZgB4FlQsKdQpCgsTFekbivDzjGHheKlMGBQ+LbZlcrys83YDOEZVgYPMf -T76cn32gsoTDV43X3cOcU9oJTDmJ5BhTBDHaAV/ctD/kqtmsj2f1K4SB2gf+tF9xdsoxD9Dpx4FF -/NN+xXVox85OkGcACqou2uKBGwCnW5/cNLLAuNp9MH7cFMAGMx8MxSKx7EUnerjz63KibdkyJRT3 -MS+fcICzKmxKmu7spqS1P3qOqwLPuZbj/kbwtk+2zGcOXW86b4aS39xPRwqxJBYw6rb2xzDZYZ2m -ejoOsw1xC21rtY39OXNipU67RYaiDEQcu50nLpP1K2HdnDnQS6PuABPfanSNJPaq8tHP2Uh7GB4m -ltidfYrpSGUsZAQwkiF17U8NPhRaBFAglP07diR3Onl+6M3RsQYPz1HrLrCNP4Ai1Lm4VOORl8CJ -8OVXdhz5FaGFevRIhI6nkskst3li+Llbo1f50p9jrwxQEBPFroyzazlmWFMD8yuf2AMhWNK2Hqkv -k6s+wyLOwDm9H+Dwrlz0H5wY1FqM0Gl3I7dtdeSTBxv0loLsJJgPvozvQPcXdTXmlRw4h+6tpRuG -+jBEzD6Epvr0fRxiOObXcGB9GsC91NCw0MP7deDsktfGOLLWPraqmkL7QnuwixK2ZpWiYxmnONH4 -otYLaAzucWPyR/apThSyv3vqxJyYkAXKg7sgvbmNdINWOGHE5UpcOZpQOnxTTaPfLeWtTMFogJEd -Y7XDL7baYRLZcEpvHthvxu5ie7Htx43eNJgdmXIMRIAKMXoDPbsQanDAFf5Z70Ti7Iac47d/PZuK -tx9+gn/fyI9gQbHmcSr+BqOLt3kJ20ou2qXbFLCAo+L9Yl4rLIwkaHRCwRdPoLd24ZEXT0N0ZYlf -UmIVpMBk2nLDt50AijxBKmRv3ANTLwG/TUFXywk1DmLfWoz0S6TBcI0L1oUc6JbRutqkaCac4Eiz -iJej87O3px8+nUbVPTK2+Tlygid+HhZORx8Nl3gMNhX2yaLGJ1eOv/yDTIsed1nvNU29DO41RQjb -kcLuL/kmjdjuKeISAwai2C7zRYQtgdO5RK+6A/954mwrH7TvnnFFWOOJPjxrnHh8DNQQP7f1zwga -Uh89J+pJCMVzrBXjx9Go3wJPBUW04c/zm7ulGxDXRT80wTamzazHfnerAtdMZw3PchLhdWyXwdSB -pkmsNvOFWx/4MRP6IhRQbnS8IVdxnVZCZrCVor093UgBCt4t6WMJYVZhK0Z1bhSdSe/irXJyj2Il -RjjqiIrq8RyGAoWw9f4xvmEzgLWGouYSaIBOiNK2KXe6qnqxZgnmnRBRryff4C7JXrnJL5rCPChv -jBeN/wrzRG+RMbqWlZ4/PxhPLl82CQ4UjF54Bb2LAoydyyZ7oDGL58+fj8S/Pez0MCpRmuc34I0B -7F5n5ZxeDxhsPTm7Wl2H3ryJgB8Xa3kJD64oaG6f1xlFJHd0pQWR9q+BEeLahJYZTfuWOeZYXcnn -y9yCz6m0wfhLltB1RxhRkqhs9a1RGG0y0kQsCYohjNUiSUKOTsB6bPMaa/Ewuqj5Rd4DxycIZopv -8WCMd9hrdCwpb9Zyj0XnWIwI8IhSyng0KmamajTAc3ax1WjOzrKkaspIXrhnpvoKgMreYqT5SsR3 -KBlmHi1iOGWdHqs2jnW+k0W9jUq+uHTjjK1Z8uuHcAfWBknLVyuDKTw0i7TIZbkw5hRXLFkklQPG -tEM43JkubyLrEwU9KI1AvZNVWFqJtm//YNfFxfQjHR/vm5F01lBlL8TimFCctfIKo6gZn6JPlpCW -b82XCYzygaLZ2hPwxhJ/0LFUrCHw7u1wyxnrTN/HwWkbzSUdAIfugLIK0rKjpyOci8csfGbagVs0 -8EM7c8LtNimrOk5n+tqHGfppM3uervG0ZXA7CzyttwK+fQ6O777O2AfHwSTXID0x49ZUZByLlY5M -RG5lmV+EVeTo5R2yrwQ+BVJmOTP10CZ2dGnZ1Raa6gRHR8UjqK9M8dKAQ26qZjoFJy7mU0pvMuUO -A86zn29JV1eI78T41VQctnY+i2KLNzkBss+Woe+KUTeYihMMMHNs34shvjsW45dT8ccd0KOBAY4O -3RHa+9gWhEEgr66eTMY0mRPZwr4U9of76hxG0PSM4+SqTf4umb4lKv1ri0pcIagTlV+2E5VbYw/u -WzsfH8lwA4pjlcjl/jOFJNRIN7p5mMEJPyyg37M5Wrp2vKmoocK5OWxG7ho96GhE4zbbQUxRulZf -XL+LuoYNp71zwKTJtFIV7S1zmMao0WsRFQDM+o7S8Bve7QLvNSlc/2zwiFUXAViwPREEXenJB2ZN -w0ZQH3QEn6QBHmAUEeJhaqMoXMl6goiEdA8OMdFXrUNsh+N/d+bhEoOho9AOlt98vQtPVzB7izp6 -FnR3pYUnsra8ollu8+kPzHmM0tf1NwmMA6URHXBWzVWV5GYeYfYy30GT2yzmDV4GSSfTaBJT6bpN -vJXmW7/Qj6HYASWTwVqAJ1Wv8CD5lu62PFGU9IZX1Hx9+HJqKoMZkJ7Aq+jVV/oKSOpmLj/wfeyp -3rvBS93vMPoXB1hS+b3tq85uhqZ13LoLyh8spOjZJJpZOjSG6eE6kGbNYoF3JjbEZN/aXgDyHryd -Ofg55vLTHBw22JBGfei6GqOR3iHVNiDAD5uMIcl5VNdGkSLSu4RtSHnuUpxPFgXdq9+CYAgBOX8d -8xt0BeviyIbYjE3Bk8+xm82Jn+qmt+6M7Qka2+om3DV97r9r7rpFYGdukhk6c/frS10a6L7DVrSP -Bhze0IR4VIlEo/H7jYlrB6Y6h6Y/Qq8/SH63E850wKw8BMZk7GC8n9hTY2/M/iZeuN8xIWyfL2R2 -y4l7nY3WtDs2o83xj/EUOPkFn9sbBiijaak5kPdLdMPejHNkZ/L6Ws1ivN1xRptsyufq7J7Mtu09 -Xc4nY7U1uy28tAhAGG7Smbducj0wBuhKvmWa06Gc22kEDU1Jw04WskqWbBL01g7ARRwxpf4mEM9p -xKNUYqBb1WVRwm54pO8i5jydvtTmBqgJ4G1idWNQNz2m+mpaUqyUHGZKkDlO20ryASKwEe+YhtnM -vgNeedFcs5BMLTPIrN7IMq6aK4b8jIAENl3NCFR0jovrhOcaqWxxiYtYYnnDQQoDZPb7V7Cx9DbV -O+5VmFht93h2oh465PuUKxscY2S4OLm31wu611ot6Wpr1zu0zRqus1cqwTKYu/JIR+pYGb/V93fx -HbMcyUf/0uEfkHe38tLPQrfqjL1bi4bzzFUI3Qub8MYAMs599zB2OKB742JrA2zH9/WFZZSOhznQ -2FJR++S9CqcZbdJEkDBh9IEIkl8U8MQIkgf/kREkfWsmGBqNj9YDvWUCD4SaWD24V1A2jAB9ZkAk -PMBuXWBoTOXYTbovcpXcj+yF0qwrnUo+Yx6QI7t3kxEIvmpSuRnK3lVwuyJIvnTR4+/PP745OSda -zC5O3v7HyfeUlIXHJS1b9egQW5bvM7X3vfRvN9ymE2n6Bm+w7bkhlmuYNITO+04OQg+E/nq1vgVt -KzL39VCHTt1PtxMgvnvaLahDKrsXcscv0zUmbvpMK0870E85qdb8cjITzCNzUsfi0JzEmffN4YmW -0U5seWjhnPTWrjrR/qq+BXQg7j2xSda0Anhmgvxlj0xMxYwNzLOD0v7ffFBmOFYbmht0QAoX0rnJ -kS5xZFCV//8TKUHZxbi3Y0dxau/mpnZ8PKTspfN49ruQkSGIV+436s7PFfalTAeoEASs8PQ9hYyI -0X/6QNWmHzxT4nKfCov3Udlc2V+4Ztq5/WuCSQaVve9LcYISH7NC41WduokDtk+nAzl9dBqVr5xK -FtB8B0DnRjwVsDf6S6wQ51sRwsZRu2SYHEt01Jf1Ocij3XSwN7R6IfaHyk7dskshXg43XLYqO3WP -Q+6hHuihalPc51hgzNIcqicV3xFkPs4UdMGX53zgGbre9sPX28uXR/ZwAfkdXzuKhLLJRo5hv3Sy -MXdeKul0J2Ypp5Suh3s1JySsW1w5UNknGNrbdEpSBvY/Js+BIY289/0hM9PDu3p/1MbUst4RTEmM -n6kJTcsp4tG42yeT7nQbtdUFwgVJjwDSUYEAC8F0dKOTILrlLO/xC70bnNd0Ha97whQ6UkHJYj5H -cA/j+zX4tbtTIfGjujOKpj83aHOgXnIQbvYduNXEC4UMm4T21Bs+GHABuCa7v//LR/TvpjHa7oe7 -/Grb6lVvHSD7spj5iplBLRKZxxEYGdCbY9LWWC5hBB2voWno6DJUMzfkC3T8KJsWL9umDQY5szPt -AVijEPwfucjncQ== -""") - -##file activate.sh -ACTIVATE_SH = convert(""" -eJytVd9v2kAMfs9fYQLq2m4MscdNVKMqEkgtVIQxbeuUHolpTgsXdHehpT/+9/mSEBJS2MOaB0ji -z77P9menDpOAK5jzEGERKw0zhFihD/dcB2CrKJYewoyLFvM0XzGNNpzOZbSAGVPBqVWHdRSDx4SI -NMhYANfgc4meDteW5ePGC45P4MkCumKhUENzDsu1H3lw1vJx1RJxGMKns6O2lWDqINGgotAHFCsu -I7FAoWHFJGezEFWGqsEvaD5C42naHb93X+A3+elYCgVaxgh8DmQAys9HL2SS0mIaWBgm7mTN/O3G -kzu6vHCng/HkW/fSve5O+hTOpnhfQAcoEry5jKVjNypoO0fgwzKSOgHm79KUK06Jfc7/RebHpD8a -9kdXvT2UcnuFWG6p0stNB0mWUUQ1q3uiGRVEMfXHR03dTuQATPjwqIIPcB9wL4CArRAY/ZHJixYL -Y9YBtcAoLQtFevOoI9QaHcEdMSAB0d08kuZhyUiSmav6CPCdVBnFOjNrLu6yMCWgKRA0TInBC5i4 -QwX3JG/mm581GKnSsSSxJTFHf9MAKr8w5T/vOv1mUurn5/zlT6fvTntjZzAaNl9rQ5JkU5KIc0GX -inagwU57T2eddqWlTrvaS6d9sImZeUMkhWysveF0m37NcGub9Dpgi0j4qGiOzATjDr06OBjOYQOo -7RBoGtNm9Denv1i0LVI7lxJDXLHSSBeWRflsyyqw7diuW3h0XdvK6lBMyaoMG1UyHdTsoYBuue75 -YOgOu1c91/2cwYpznPPeDoQpGL2xSm09NKp7BsvQ2hnT3aMs07lUnskpxewvBk73/LLnXo9HV9eT -ijB3hWBO2ygoiWg/bKuZxqCCQq0DD3vkWIVvI2KosIw+vqW1gIItEG5KJb+xb09g65ktwYKgTc51 -uGJ/EFQs0ayEWLCQM5V9N4g+1+8UbXOJzF8bqhKtIqIwicWvzNFROZJlpfD8A7Vc044R0FxkcezG -VzsV75usvTdYef+57v5n1b225qhXfwEmxHEs -""") - -##file activate.fish -ACTIVATE_FISH = convert(""" -eJyFVVFv2zYQftevuMoOnBS1gr0WGIZ08RADSRw4boBhGGhGOsUcKFIjKbUu9uN7lC2JsrXWDzZM -fnf38e6+uwlsdsJCLiRCUVkHrwiVxYy+hHqDbQKvQl3z1ImaO0xyYXdbeP9FuJ1QwMFUSnmcP4dL -2DlXfry+9v/sDqVMUl3AFVi0Vmj1PokmcKtBaecNQTjIhMHUyX0SRXmlKIpWkGEbDuYZzBZfCVcL -4youUdVQ6AyBqwwMusoocBrcDsmpKbgEQgijVYHKJbMI6DMhoEUHWmbhLdTcCP4q0TYokYNDev5c -QTxlq/tb9rJcbz7f3LOnm81d3GD8x3uav30FfwrnwCEOYRyAKot+FvXPzd3q8W71sBiJ3d2dMugu -fsxjCPsBmz+Wz3fsab16eNqw1ctivV7eBnwm8EzeuQIsSrcHqVMqwHbqq8/aarKSO+oYKhKXUn9p -SmWw0DVBdQ7bBlwaTR62bc+1tpaYb5PhUyScu48CRgvDLQbtMrMnMQ6dY5022JDRRrwJxWUfJwwP -ge0YIAVGfcUC1M8s8MxitFZjmR9W64hui7p4fBlWMZ5y81b/9cvfMbz7FWZKq4yOTeW1hbNBEWU+ -b+/ejXMu95lOx696uXb8Go4T+Kw8R2EMSqx5KLkkCkQ+ZBZFbZsHL4OYseAvY3EPO5MYTBuhDZQa -TwPza8Y+LR/Z483Dgjwd4R3f7bTXx9Znkw6T6PAL83/hRD3jNAKFjuEx9NJkq5t+fabLvdvRwbw4 -nEFTzwO6U+q34cvY7fL55tP94tg58XEA/q7LfdPsaUXFoEIMJdHF5iSW0+48CnDQ82G7n3XzAD6q -Bmo5XuOA0NQ67ir7AXJtQhtLKO7XhC0l39PGOBsHPvzBuHUSjoOnA0ldozGC9gZ5rek3+y3ALHO/ -kT7AP379lQZLSnFDLtwWihfYxw4nZd+ZR7myfkI2ZTRCuRxmF/bCzkbhcElvYamW9PbDGrvqPKC0 -+D/uLi/sFcxGjOHylYagZzzsjjhw206RQwrWIwOxS2dnk+40xOjX8bTPegz/gdWVSXuaowNuOLda -wYyNuRPSTcd/B48Ppeg= -""") - -##file activate.csh -ACTIVATE_CSH = convert(""" -eJx1U2FP2zAQ/e5f8TAV3Soo+0zXbYUiDQkKQgVp2ibjJNfFUuIg22nVf885SVFLO3+I7Lt3fr6X -d8eY58ZjYQpCWfuAhFB7yrAyIYf0Ve1SQmLsuU6DWepAw9TnEoOFq0rwdjAUx/hV1Ui1tVWAqy1M -QGYcpaFYx+yVI67LkKwx1UuTEaYGl4X2Bl+zJpAlP/6V2hTDtCq/DYXQhdEeGW040Q/Eb+t9V/e3 -U/V88zh/mtyqh8n8J47G+IKTE3gKZJdoYrK3h5MRU1tGYS83gqNc+3yEgyyP93cP820evHLvr2H8 -kaYB/peoyY7aVHzpJnE9e+6I5Z+ji4GMTNJWNuOQq6MA1N25p8pW9HWdVWlfsNpPDbdxjgpaahuw -1M7opCA/FFu1uwxC7L8KUqmto1KyQe3rx0I0Eovdf7BVe67U5c1MzSZ310pddGheZoFPWyytRkzU -aCA/I+RkBXhFXr5aWV0SxjhUI6jwdAj8kmhPzX7nTfJFkM3MImp2VdVFFq1vLHSU5szYQK4Ri+Jd -xlW2JBtOGcyYVW7SnB3v6RS91g3gKapZ0oWxbHVteYIIq3iv7QeuSrUj6KSqQ+yqsxDj1ivNQxKF -YON10Q+NH/ARS95i5Tuqq2Vxfvc23f/FO6zrtXXmJr+ZtMY9/A15ZXFWtmch2rEQ4g1ryVHH -""") - -##file activate.bat -ACTIVATE_BAT = convert(""" -eJx9Ul9LhEAQfxf8DoOclI/dYyFkaCmcq4gZQTBUrincuZFbff12T133TM+nnd35/Zvxlr7XDFhV -mUZHOVhFlOWP3g4DUriIWoVomYZpNBWUtGpaWgImO191pFkSpzlcmgaI70jVX7n2Qp8tuByg+46O -CMHbMq64T+nmlJt082D1T44muCDk2prgEHF4mdI9RaS/QwSt3zSyIAaftRccvqVTBziD1x/WlPD5 -xd729NDBb8Nr4DU9QNMKsJeH9pkhPedhQsIkDuCDCa6A+NF9IevVFAohkqizdHetg/tkWvPoftWJ -MCqnOxv7/x7Np6yv9P2Ker5dmX8yNyCkkWnbZy3N5LarczlqL8htx2EM9rQ/2H5BvIsIEi8OEG8U -+g8CsNTr -""") - -##file deactivate.bat -DEACTIVATE_BAT = convert(""" -eJyFkN0KgkAUhO8F32EQpHqFQEjQUPAPMaErqVxzId3IrV6/XST/UDx3c86c4WMO5FYysKJQFVVp -CEfqxsnJ9DI7SA25i20fFqs3HO+GYLsDZ7h8GM3xfLHrg1QNvpSX4CWpQGvokZk4uqrQAjXjyElB -a5IjCz0r+2dHcehHCe5MZNmB5R7TdqMqECMptHZh6DN/utb7Zs6Cej8OXYE5J04YOKFvD4GkHuJ0 -pilSd1jG6n87tDZ+BUwUOepI6CGSkFMYWf0ihvT33Qj1A+tCkSI= -""") - -##file activate.ps1 -ACTIVATE_PS = convert(""" -eJylWdmO41hyfW+g/0FTU7C7IXeJIqmtB/3AnZRIStxF2kaBm7gv4ipyMF/mB3+Sf8GXVGVl1tLT -43ECSqR4b5wbETeWE8z/+a///vNCDaN6cYtSf5G1dbNw/IVXNIu6aCvX9xa3qsgWl0IJ/7IYinbh -2nkOVqs2X0TNjz/8eeFFle826fBhQRaLBkD9uviw+LCy3Sbq7Mb/UNbrH3+YNtLcVaB+Xbipb+eL -tly0eVsD/M6u6g8//vC+dquobH5VWU75eMFUdvHb4n02RHlXuHYTFfmHbHCLLLNz70NpN+GrBI4p -1EeSk4FAXaZR88u0vPip8usi7fznt3fvP+OuPnx49/Pil4td+XnzigIAPoqYQH2J8v4z+C+8b98m -Q25t7k76LIK0cOz0V89/MXXx0+Lf6z5q3PA/F+/FIif9uqnaadFf/PzXSXYBfqIb2NeApecJwPzI -dlL/149nnvyoc7KqYfzTAT8v/voUmX7e+3n364tffl/oVaDyswKY/7J18e6bve8Wv9RuUfqfLHmK -/u139Hwx+9ePRep97KKqae30YwmCo2y+0vTz1k+rv7159B3pb1SOGj97Pe8/flfkC1Vn/7xYR4n6 -lypNEGDDV5f7lcjil3S+4++p881Wv6qKyn5GQg1yJwcp4BZ5E+Wt/z1P/umbiHir4J8Xip/eFt6n -9T/9gU9eY+7zUX97Jlmb136ziKrKT/3OzpvP8VX/+MObSP0lL3LvVZlJ9v1b8357jXyw8rXxYPXN -11n4UzJ8G8S/vUbuJ6RPj999DbtS5kys//JusXwrNLnvT99cFlBNwXCe+niRz8JF/ezNr9Pze+H6 -18W7d5PPvozW7+387Zto/v4pL8BvbxTzvIW9KCv/Fj0WzVQb/YXbVlPZWTz3/9vCaRtQbPN/Bb+j -2rUrDxTVD68gfQXu/ZewAFX53U/vf/rD2P3558W7+W79Po1y/xXoX/6RFHyNIoVjgAG4H0RTcAe5 -3bSVv3DSwk2mZYHjFB8zj6fC4sLOFTHJJQrwzFYJgso0ApOoBzFiRzzQKjIQCCbQMIFJGCKqGUyS -8AkjiF2wTwmMEbcEUvq8Nj+X0f4YcCQmYRiOY7eRbAJDqzm1chOoNstbJ8oTBhZQ2NcfgaB6QjLp -U4+SWFjQGCZpyqby8V4JkPGs9eH1BscXIrTG24QxXLIgCLYNsIlxSYLA6SjAeg7HAg4/kpiIB8k9 -TCLm0EM4gKIxEj8IUj2dQeqSxEwYVH88qiRlCLjEYGuNIkJB1BA5dHOZdGAoUFk54WOqEojkuf4Q -Ig3WY+96TDlKLicMC04h0+gDCdYHj0kz2xBDj9ECDU5zJ0tba6RKgXBneewhBG/xJ5m5FX+WSzsn -wnHvKhcOciw9NunZ0BUF0n0IJAcJMdcLqgQb0zP19dl8t9PzmMBjkuIF7KkvHgqEovUPOsY0PBB1 -HCtUUhch83qEJPjQcNQDsgj0cRqx2ZbnnlrlUjE1EX2wFJyyDa/0GLrmKDEFepdWlsbmVU45Wiwt -eFM6mfs4kxg8yc4YmKDy67dniLV5FUeO5AKNPZaOQQ++gh+dXE7dbJ1aTDr7S4WPd8sQoQkDyODg -XnEu/voeKRAXZxB/e2xaJ4LTFLPYEJ15Ltb87I45l+P6OGFA5F5Ix8A4ORV6M1NH1uMuZMnmFtLi -VpYed+gSq9JDBoHc05J4OhKetrk1p0LYiKipxLMe3tYS7c5V7O1KcPU8BJGdLfcswhoFCSGQqJ8f -ThyQKy5EWFtHVuNhvTnkeTc8JMpN5li3buURh0+3ZGuzdwM55kon+8urbintjdQJf9U1D0ah+hNh -i1XNu4fSKbTC5AikGEaj0CYM1dpuli7EoqUt7929f1plxGGNZnixFSFP2qzhlZMonu2bB9OWSqYx -VuHKWNGJI8kqUhMTRtk0vJ5ycZ60JlodlmN3D9XiEj/cG2lSt+WV3OtMgt1Tf4/Z+1BaCus740kx -Nvj78+jMd9tq537Xz/mNFyiHb0HdwHytJ3uQUzKkYhK7wjGtx3oKX43YeYoJVtqDSrCnQFzMemCS -2bPSvP+M4yZFi/iZhAjL4UOeMfa7Ex8HKBqw4umOCPh+imOP6yVTwG2MplB+wtg97olEtykNZ6wg -FJBNXSTJ3g0CCTEEMdUjjcaBDjhJ9fyINXgQVHhA0bjk9lhhhhOGzcqQSxYdj3iIN2xGEOODx4qj -Q2xikJudC1ujCVOtiRwhga5nPdhe1gSa649bLJ0wCuLMcEYIeSy25YcDQHJb95nfowv3rQnin0fE -zIXFkM/EwSGxvCCMgEPNcDp/wph1gMEa8Xd1qAWOwWZ/KhjlqzgisBpDDDXz9Cmov46GYBKHC4zZ -84HJnXoTxyWNBbXV4LK/r+OEwSN45zBp7Cub3gIYIvYlxon5BzDgtPUYfXAMPbENGrI+YVGSeTQ5 -i8NMB5UCcC+YRGIBhgs0xhAGwSgYwywpbu4vpCSTdEKrsy8osXMUnHQYenQHbOBofLCNNTg3CRRj -A1nXY2MZcjnXI+oQ2Zk+561H4CqoW61tbPKv65Y7fqc3TDUF9CA3F3gM0e0JQ0TPADJFJXVzphpr -2FzwAY8apGCju1QGOiUVO5KV6/hKbtgVN6hRVwpRYtu+/OC6w2bCcGzZQ8NCc4WejNEjFxOIgR3o -QqR1ZK0IaUxZ9nbL7GWJIjxBARUhAMnYrq/S0tVOjzlOSYRqeIZxaSaOBX5HSR3MFekOXVdUPbjX -nru61fDwI8HRYPUS7a6Inzq9JLjokU6P6OzT4UCH+Nha+JrU4VqEo4rRHQJhVuulAnvFhYz5NWFT -aS/bKxW6J3e46y4PLagGrCDKcq5B9EmP+s1QMCaxHNeM7deGEV3WPn3CeKjndlygdPyoIcNaL3dd -bdqPs47frcZ3aNWQ2Tk+rjFR01Ul4XnQQB6CSKA+cZusD0CP3F2Ph0e78baybgioepG12luSpFXi -bHbI6rGLDsGEodMObDG7uyxfCeU+1OiyXYk8fnGu0SpbpRoEuWdSUlNi5bd9nBxYqZGrq7Qa7zV+ -VLazLcelzzP9+n6+xUtWx9OVJZW3gk92XGGkstTJ/LreFVFF2feLpXGGuQqq6/1QbWPyhJXIXIMs -7ySVlzMYqoPmnmrobbeauMIxrCr3sM+qs5HpwmmFt7SM3aRNQWpCrmeAXY28EJ9uc966urGKBL9H -18MtDE5OX97GDOHxam11y5LCAzcwtkUu8wqWI1dWgHyxGZdY8mC3lXzbzncLZ2bIUxTD2yW7l9eY -gBUo7uj02ZI3ydUViL7oAVFag37JsjYG8o4Csc5R7SeONGF8yZP+7xxi9scnHvHPcogJ44VH/LMc -Yu6Vn3jEzCFw9Eqq1ENQAW8aqbUwSiAqi+nZ+OkZJKpBL66Bj8z+ATqb/8qDIJUeNRTwrI0YrVmb -9FArKVEbCWUNSi8ipfVv+STgkpSsUhcBg541eeKLoBpLGaiHTNoK0r4nn3tZqrcIULtq20Df+FVQ -Sa0MnWxTugMuzD410sQygF4qdntbswiJMqjs014Irz/tm+pd5oygJ0fcdNbMg165Pqi7EkYGAXcB -dwxioCDA3+BY9+JjuOmJu/xyX2GJtaKSQcOZxyqFzTaa6/ot21sez0BtKjirROKRm2zuai02L0N+ -ULaX8H5P6VwsGPbYOY7sAy5FHBROMrMzFVPYhFHZ7M3ZCZa2hsT4jGow6TGtG8Nje9405uMUjdF4 -PtKQjw6yZOmPUmO8LjFWS4aPCfE011N+l3EdYq09O3iQJ9a01B3KXiMF1WmtZ+l1gmyJ/ibAHZil -vQzdOl6g9PoSJ4TM4ghTnTndEVMOmsSSu+SCVlGCOLQRaw9oLzamSWP62VuxPZ77mZYdfTRGuNBi -KyhZL32S2YckO/tU7y4Bf+QKKibQSKCTDWPUwWaE8yCBeL5FjpbQuAlb53mGX1jptLeRotREbx96 -gnicYz0496dYauCjpTCA4VA0cdLJewzRmZeTwuXWD0talJsSF9J1Pe72nkaHSpULgNeK1+o+9yi0 -YpYwXZyvaZatK2eL0U0ZY6ekZkFPdC8JTF4Yo1ytawNfepqUKEhwznp6HO6+2l7L2R9Q3N49JMIe -Z+ax1mVaWussz98QbNTRPo1xu4W33LJpd9H14dd66ype7UktfEDi3oUTccJ4nODjwBKFxS7lYWiq -XoHu/b7ZVcK5TbRD0F/2GShg2ywwUl07k4LLqhofKxFBNd1grWY+Zt/cPtacBpV9ys2z1moMLrT3 -W0Elrjtt5y/dvDQYtObYS97pqj0eqmwvD3jCPRqamGthLiF0XkgB6IdHLBBwDGPiIDh7oPaRmTrN -tYA/yQKFxRiok+jM6ciJq/ZgiOi5+W4DEmufPEubeSuYJaM3/JHEevM08yJAXUQwb9LS2+8FOfds -FfOe3Bel6EDSjIEIKs4o9tyt67L1ylQlzhe0Q+7ue/bJnWMcD3q6wDSIQi8ThnRM65aqLWesi/ZM -xhHmQvfKBbWcC194IPjbBLYR9JTPITbzwRcu+OSFHDHNSYCLt29sAHO6Gf0h/2UO9Xwvhrjhczyx -Ygz6CqP4IwxQj5694Q1Pe2IR+KF/yy+5PvCL/vgwv5mPp9n4kx7fnY/nmV++410qF/ZVCMyv5nAP -pkeOSce53yJ6ahF4aMJi52by1HcCj9mDT5i+7TF6RoPaLL+cN1hXem2DmX/mdIbeeqwQOLD5lKO/ -6FM4x77w6D5wMx3g0IAfa2D/pgY9a7bFQbinLDPz5dZi9ATIrd0cB5xfC0BfCCZO7TKP0jQ2Meih -nRXhkA3smTAnDN9IW2vA++lsgNuZ2QP0UhqyjUPrDmgfWP2bWWiKA+YiEK7xou8cY0+d3/bk0oHR -QLrq4KzDYF/ljQDmNhBHtkVNuoDey6TTeaD3SHO/Bf4d3IwGdqQp6FuhmwFbmbQBssDXVKDBYOpk -Jy7wxOaSRwr0rDmGbsFdCM+7XU/84JPu3D/gW7QXgzlvbjixn99/8CpWFUQWHFEz/RyXvzNXTTOd -OXLNNFc957Jn/YikNzEpUdRNxXcC6b76ccTwMGoKj5X7c7TvHFgc3Tf4892+5A+iR+D8OaaE6ACe -gdgHcyCoPm/xiDCWP+OZRjpzfj5/2u0i4qQfmIEOsTV9Hw6jZ3Agnh6hiwjDtGYxWvt5TiWEuabN -77YCyRXwO8P8wdzG/8489KwfFBZWI6Vvx76gmlOc03JI1HEfXYZEL4sNFQ3+bqf7e2hdSWQknwKF -ICJjGyDs3fdmnnxubKXebpQYLjPgEt9GTzKkUgTvOoQa1J7N3nv4sR6uvYFLhkXZ+pbCoU3K9bfq -gF7W82tNutRRZExad+k4GYYsCfmEbvizS4jsRr3fdzqjEthpEwm7pmN7OgVzRbrktjrFw1lc0vM8 -V7dyTJ71qlsd7v3KhmHzeJB35pqEOk2pEe5uPeCToNkmedmxcKbIj+MZzjFSsvCmimaMQB1uJJKa -+hoWUi7aEFLvIxKxJavqpggXBIk2hr0608dIgnfG5ZEprqmH0b0YSy6jVXTCuIB+WER4d5BPVy9Q -M4taX0RIlDYxQ2CjBuq78AAcHQf5qoKP8BXHnDnd/+ed5fS+csL4g3eWqECaL+8suy9r8hx7c+4L -EegEWdqAWN1w1NezP34xsxLkvRRI0DRzKOg0U+BKfQY128YlYsbwSczEg2LqKxRmcgiwHdhc9MQJ -IwKQHlgBejWeMGDYYxTOQUiJOmIjJbzIzHH6lAMP+y/fR0v1g4wx4St8fcqTt3gz5wc+xXFZZ3qI -JpXI5iJk7xmNL2tYsDpcqu0375Snd5EKsIvg8u5szTOyZ4v06Ny2TZXRpHUSinh4IFp8Eoi7GINJ -02lPJnS/9jSxolJwp2slPMIEbjleWw3eec4XaetyEnSSqTPRZ9fVA0cPXMqzrPYQQyrRux3LaAh1 -wujbgcObg1nt4iiJ5IMbc/WNPc280I2T4nTkdwG8H6iS5xO2WfsFsruBwf2QkgZlb6w7om2G65Lr -r2Gl4dk63F8rCEHoUJ3fW+pU2Srjlmcbp+JXY3DMifEI22HcHAvT7zzXiMTr7VbUR5a2lZtJkk4k -1heZZFdru8ucCWMTr3Z4eNnjLm7LW7rcN7QjMpxrsCzjxndeyFUX7deIs3PQkgyH8k6luI0uUyLr -va47TBjM4JmNHFzGPcP6BV6cYgQy8VQYZe5GmzZHMxyBYhGiUdekZQ/qwyxC3WGylQGdUpSf9ZCP -a7qPdJd31fPRC0TOgzupO7nLuBGr2A02yuUQwt2KQG31sW8Gd9tQiHq+hPDt4OzJuY4pS8XRsepY -tsd7dVEfJFmc15IYqwHverrpWyS1rFZibDPW1hUUb+85CGUzSBSTK8hpvee/ZxonW51TUXekMy3L -uy25tMTg4mqbSLQQJ+skiQu2toIfBFYrOWql+EQipgfT15P1aq6FDK3xgSjIGWde0BPftYchDTdM -i4QdudHFkN0u6fSKiT09QLv2mtSblt5nNzBR6UReePNs+khE4rHcXuoK21igUKHl1c3MXMgPu7y8 -rKQDxR6N/rffXv+lROXet/9Q+l9I4D1U -""") - -##file distutils-init.py -DISTUTILS_INIT = convert(""" -eJytV1uL4zYUfvevOE0ottuMW9q3gVDa3aUMXXbLMlDKMBiNrSTqOJKRlMxkf33PkXyRbGe7Dw2E -UXTu37lpxLFV2oIyifAncxmOL0xLIfcG+gv80x9VW6maw7o/CANSWWBwFtqeWMPlGY6qPjV8A0bB -C4eKSTgZ5LRgFeyErMEeOBhbN+Ipgeizhjtnhkn7DdyjuNLPoCS0l/ayQTG0djwZC08cLXozeMss -aG5EzQ0IScpnWtHSTXuxByV/QCmxE7y+eS0uxWeoheaVVfqSJHiU7Mhhi6gULbOHorshkrEnKxpT -0n3A8Y8SMpuwZx6aoix3ouFlmW8gHRSkeSJ2g7hU+kiHLDaQw3bmRDaTGfTnty7gPm0FHbIBg9U9 -oh1kZzAFLaue2R6htPCtAda2nGlDSUJ4PZBgCJBGVcwKTAMz/vJiLD+Oin5Z5QlvDPdulC6EsiyE -NFzb7McNTKJzbJqzphx92VKRFY1idenzmq3K0emRcbWBD0ryqc4NZGmKOOOX9Pz5x+/l27tP797c -f/z0d+4NruGNai8uAM0bfsYaw8itFk8ny41jsfpyO+BWlpqfhcG4yxLdi/0tQqoT4a8Vby382mt8 -p7XSo7aWGdPBc+b6utaBmCQ7rQKQoWtAuthQCiold2KfJIPTT8xwg9blPumc+YDZC/wYGdAyHpJk -vUbHbHWAp5No6pK/WhhLEWrFjUwtPEv1Agf8YmnsuXUQYkeZoHm8ogP16gt2uHoxcEMdf2C6pmbw -hUMsWGhanboh4IzzmsIpWs134jVPqD/c74bZHdY69UKKSn/+KfVhxLgUlToemayLMYQOqfEC61bh -cbhwaqoGUzIyZRFHPmau5juaWqwRn3mpWmoEA5nhzS5gog/5jbcFQqOZvmBasZtwYlG93k5GEiyw -buHhMWLjDarEGpMGB2LFs5nIJkhp/nUmZneFaRth++lieJtHepIvKgx6PJqIlD9X2j6pG1i9x3pZ -5bHuCPFiirGHeO7McvoXkz786GaKVzC9DSpnOxJdc4xm6NSVq7lNEnKdVlnpu9BNYoKX2Iq3wvgh -gGEUM66kK6j4NiyoneuPLSwaCWDxczgaolEWpiMyDVDb7dNuLAbriL8ig8mmeju31oNvQdpnvEPC -1vAXbWacGRVrGt/uXN/gU0CDDwgooKRrHfTBb1/s9lYZ8ZqOBU0yLvpuP6+K9hLFsvIjeNhBi0KL -MlOuWRn3FRwx5oHXjl0YImUx0+gLzjGchrgzca026ETmYJzPD+IpuKzNi8AFn048Thd63OdD86M6 -84zE8yQm0VqXdbbgvub2pKVnS76icBGdeTHHXTKspUmr4NYo/furFLKiMdQzFjHJNcdAnMhltBJK -0/IKX3DVFqvPJ2dLE7bDBkH0l/PJ29074+F0CsGYOxsb7U3myTUncYfXqnLLfa6sJybX4g+hmcjO -kMRBfA1JellfRRKJcyRpxdS4rIl6FdmQCWjo/o9Qz7yKffoP4JHjOvABcRn4CZIT2RH4jnxmfpVG -qgLaAvQBNfuO6X0/Ux02nb4FKx3vgP+XnkX0QW9pLy/NsXgdN24dD3LxO2Nwil7Zlc1dqtP3d7/h -kzp1/+7hGBuY4pk0XD/0Ao/oTe/XGrfyM773aB7iUhgkpy+dwAMalxMP0DrBcsVw/6p25+/hobP9 -GBknrWExDhLJ1bwt1NcCNblaFbMKCyvmX0PeRaQ= -""") - -##file distutils.cfg -DISTUTILS_CFG = convert(""" -eJxNj00KwkAMhfc9xYNuxe4Ft57AjYiUtDO1wXSmNJnK3N5pdSEEAu8nH6lxHVlRhtDHMPATA4uH -xJ4EFmGbvfJiicSHFRzUSISMY6hq3GLCRLnIvSTnEefN0FIjw5tF0Hkk9Q5dRunBsVoyFi24aaLg -9FDOlL0FPGluf4QjcInLlxd6f6rqkgPu/5nHLg0cXCscXoozRrP51DRT3j9QNl99AP53T2Q= -""") - -##file activate_this.py -ACTIVATE_THIS = convert(""" -eJyNU01v2zAMvetXEB4K21jnDOstQA4dMGCHbeihlyEIDMWmE62yJEiKE//7kXKdpEWLzYBt8evx -kRSzLPs6wiEoswM8YdMpjUXcq1Dz6RZa1cSiTkJdr86GsoTRHuCotBayiWqQEYGtMCgfD1KjGYBe -5a3p0cRKiEe2NtLAFikftnDco0ko/SFEVgEZ8aRCZDIPY9xbA8pE9M4jfW/B2CjiHq9zbJVZuOQq -siwTIvpxKYCembPAU4Muwi/Z4zfvrZ/MXipKeB8C+qisSZYiWfjJfs+0/MFMdWn1hJcO5U7G/SLa -xVx8zU6VG/PXLXvfsyyzUqjeWR8hjGE+2iCE1W1tQ82hsCJN9dzKaoexyB/uH79TnjwvxcW0ntSb -yZ8jq1Z5Q1UXsyy3gf9nbjTEj7NzQMfCJa/YSmrQ+2D/BqfiOi6sclrGzvoeVivIj8rcfcmnIQRF -7XCyeZI7DFe5/lhlCs5PRf5QW66VXT/NrlQ46oD/D6InkOmi3IQcbhKxAX2g4a+Xd5s3UtCtG2py -m8eg6WYWqR6SL5OjKMGfSrYt/6kxxQtOpeAgj1LXBNmpE2ElmCSIy5H0zFd8gJ924HWijWhb2hRC -6wNEm1QdDZtuSZcEprIUBo/XRNcbQe1OUbQ/r3hPTaPJJDNtFLu8KHV5XoNr3Eo6h6YtOKw8e8yw -VF5PnJ+ts3a9/Mz38RpG/AUSzYUW -""") - -##file python-config -PYTHON_CONFIG = convert(""" -eJyNVV1P2zAUfc+v8ODBiSABxlulTipbO6p1LWqBgVhlhcZpPYUkctzSivHfd6+dpGloGH2Ja/ue -e+65Hz78xNhtf3x90xmw7vCWsRPGLvpDNuz87MKfdKMWSWxZ4ilNpCLZJiuWc66SVFUOZkkcirll -rfxIBAzOMtImDzSVPBRrekwoX/OZu/0r4lm0DHiG60g86u8sjPw5rCyy86NRkB8QuuBRSqfAKESn -3orLTCQxE3GYkC9tYp8fk89OSwNsmXgizrhUtnumeSgeo5GbLUMk49Rv+2nK48Cm/qMwfp333J2/ -dVcAGE0CIQHBsgIeEr4Wij0LtWDLzJ9ze5YEvH2WI6CHTAVcSu9ZCsXtgxu81CIvp6/k4eXsdfo7 -PvDCRD75yi41QitfzlcPp1OI7i/1/iQitqnr0iMgQ+A6wa+IKwwdxyk9IiXNAzgquTFU8NIxAVjM -osm1Zz526e+shQ4hKRVci69nPC3Kw4NQEmkQ65E7OodxorSvxjvpBjQHDmWFIQ1mlmzlS5vedseT -/mgIEsMJ7Lxz2bLAF9M5xeLEhdbHxpWOw0GdkJApMVBRF1y+a0z3c9WZPAXGFcFrJgCIB+024uad -0CrzmEoRa3Ub4swNIHPGf7QDV+2uj2OiFWsChgCwjKqN6rp5izpbH6Wc1O1TclQTP/XVwi6anTr1 -1sbubjZLI1+VptPSdCfwnFBrB1jvebrTA9uUhU2/9gad7xPqeFkaQcnnLbCViZK8d7R1kxzFrIJV -8EaLYmKYpvGVkig+3C5HCXbM1jGCGekiM2pRCVPyRyXYdPf6kcbWEQ36F5V4Gq9N7icNNw+JHwRE -LTgxRXACpvnQv/PuT0xCCAywY/K4hE6Now2qDwaSE5FB+1agsoUveYDepS83qFcF1NufvULD3fTl -g6Hgf7WBt6lzMeiyyWVn3P1WVbwaczHmTzE9A5SyItTVgFYyvs/L/fXlaNgbw8v3azT+0eikVlWD -/vBHbzQumP23uBCjsYdrL9OWARwxs/nuLOzeXbPJTa/Xv6sUmQir5pC1YRLz3eA+CD8Z0XpcW8v9 -MZWF36ryyXXf3yBIz6nzqz8Muyz0m5Qj7OexfYo/Ph3LqvkHUg7AuA== -""") - -MH_MAGIC = 0xfeedface -MH_CIGAM = 0xcefaedfe -MH_MAGIC_64 = 0xfeedfacf -MH_CIGAM_64 = 0xcffaedfe -FAT_MAGIC = 0xcafebabe -BIG_ENDIAN = '>' -LITTLE_ENDIAN = '<' -LC_LOAD_DYLIB = 0xc -maxint = majver == 3 and getattr(sys, 'maxsize') or getattr(sys, 'maxint') - - -class fileview(object): - """ - A proxy for file-like objects that exposes a given view of a file. - Modified from macholib. - """ - - def __init__(self, fileobj, start=0, size=maxint): - if isinstance(fileobj, fileview): - self._fileobj = fileobj._fileobj - else: - self._fileobj = fileobj - self._start = start - self._end = start + size - self._pos = 0 - - def __repr__(self): - return '' % ( - self._start, self._end, self._fileobj) - - def tell(self): - return self._pos - - def _checkwindow(self, seekto, op): - if not (self._start <= seekto <= self._end): - raise IOError("%s to offset %d is outside window [%d, %d]" % ( - op, seekto, self._start, self._end)) - - def seek(self, offset, whence=0): - seekto = offset - if whence == os.SEEK_SET: - seekto += self._start - elif whence == os.SEEK_CUR: - seekto += self._start + self._pos - elif whence == os.SEEK_END: - seekto += self._end - else: - raise IOError("Invalid whence argument to seek: %r" % (whence,)) - self._checkwindow(seekto, 'seek') - self._fileobj.seek(seekto) - self._pos = seekto - self._start - - def write(self, bytes): - here = self._start + self._pos - self._checkwindow(here, 'write') - self._checkwindow(here + len(bytes), 'write') - self._fileobj.seek(here, os.SEEK_SET) - self._fileobj.write(bytes) - self._pos += len(bytes) - - def read(self, size=maxint): - assert size >= 0 - here = self._start + self._pos - self._checkwindow(here, 'read') - size = min(size, self._end - here) - self._fileobj.seek(here, os.SEEK_SET) - bytes = self._fileobj.read(size) - self._pos += len(bytes) - return bytes - - -def read_data(file, endian, num=1): - """ - Read a given number of 32-bits unsigned integers from the given file - with the given endianness. - """ - res = struct.unpack(endian + 'L' * num, file.read(num * 4)) - if len(res) == 1: - return res[0] - return res - - -def mach_o_change(path, what, value): - """ - Replace a given name (what) in any LC_LOAD_DYLIB command found in - the given binary with a new name (value), provided it's shorter. - """ - - def do_macho(file, bits, endian): - # Read Mach-O header (the magic number is assumed read by the caller) - cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags = read_data(file, endian, 6) - # 64-bits header has one more field. - if bits == 64: - read_data(file, endian) - # The header is followed by ncmds commands - for n in range(ncmds): - where = file.tell() - # Read command header - cmd, cmdsize = read_data(file, endian, 2) - if cmd == LC_LOAD_DYLIB: - # The first data field in LC_LOAD_DYLIB commands is the - # offset of the name, starting from the beginning of the - # command. - name_offset = read_data(file, endian) - file.seek(where + name_offset, os.SEEK_SET) - # Read the NUL terminated string - load = file.read(cmdsize - name_offset).decode() - load = load[:load.index('\0')] - # If the string is what is being replaced, overwrite it. - if load == what: - file.seek(where + name_offset, os.SEEK_SET) - file.write(value.encode() + '\0'.encode()) - # Seek to the next command - file.seek(where + cmdsize, os.SEEK_SET) - - def do_file(file, offset=0, size=maxint): - file = fileview(file, offset, size) - # Read magic number - magic = read_data(file, BIG_ENDIAN) - if magic == FAT_MAGIC: - # Fat binaries contain nfat_arch Mach-O binaries - nfat_arch = read_data(file, BIG_ENDIAN) - for n in range(nfat_arch): - # Read arch header - cputype, cpusubtype, offset, size, align = read_data(file, BIG_ENDIAN, 5) - do_file(file, offset, size) - elif magic == MH_MAGIC: - do_macho(file, 32, BIG_ENDIAN) - elif magic == MH_CIGAM: - do_macho(file, 32, LITTLE_ENDIAN) - elif magic == MH_MAGIC_64: - do_macho(file, 64, BIG_ENDIAN) - elif magic == MH_CIGAM_64: - do_macho(file, 64, LITTLE_ENDIAN) - - assert(len(what) >= len(value)) - - with open(path, 'r+b') as f: - do_file(f) - - -if __name__ == '__main__': - main() - -# TODO: -# Copy python.exe.manifest -# Monkeypatch distutils.sysconfig diff --git a/thirdparty/virtualenv.py.ABOUT b/thirdparty/virtualenv.py.ABOUT deleted file mode 100644 index c91aea0e..00000000 --- a/thirdparty/virtualenv.py.ABOUT +++ /dev/null @@ -1,23 +0,0 @@ -about_resource: virtualenv.py -name: virtualenv -version: 16.0.0 -attribute: yes -checksum_md5: e90c0347ea7779ad01fb730f0e0f7845 -checksum_sha1: 8b84034c7e4d825dd8d11e44d61437bb0b05bb6d -copyright: | - Copyright (c) 2007 Ian Bicking and Contributors - Copyright (c) 2009 Ian Bicking, The Open Planning Project - Copyright (c) The virtualenv developers -description: | - virtualenv is a tool to create isolated Python environments. -download_url: https://raw.githubusercontent.com/pypa/virtualenv/16.0.0/virtualenv.py -homepage_url: http://virtualenv.org/ -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: virtualenv.py.NOTICE -owner: The virtualenv developers -owner_url: http://www.virtualenv.org/en/latest/ diff --git a/thirdparty/virtualenv.py.NOTICE b/thirdparty/virtualenv.py.NOTICE deleted file mode 100644 index 08dd0f29..00000000 --- a/thirdparty/virtualenv.py.NOTICE +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2007 Ian Bicking and Contributors -Copyright (c) 2009 Ian Bicking, The Open Planning Project -Copyright (c) 2011-2016 The virtualenv developers - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/thirdparty/wheel-0.31.1-py2.py3-none-any.whl b/thirdparty/wheel-0.31.1-py2.py3-none-any.whl deleted file mode 100644 index cfd7b906..00000000 Binary files a/thirdparty/wheel-0.31.1-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/wheel-0.31.1-py2.py3-none-any.whl.ABOUT b/thirdparty/wheel-0.31.1-py2.py3-none-any.whl.ABOUT deleted file mode 100644 index 85f320d6..00000000 --- a/thirdparty/wheel-0.31.1-py2.py3-none-any.whl.ABOUT +++ /dev/null @@ -1,21 +0,0 @@ -about_resource: wheel-0.31.1-py2.py3-none-any.whl -name: Wheel -version: 0.31.1 -checksum_md5: c6fad0d4eb25cfb29f986a5558479dc1 -checksum_sha1: c6f78e969470216bca108cd961e2380fea23613f -contact: distutils-sig@python.org -copyright: | - copyright (c) 2012-2014 Daniel Holth and contributors. -download_url: https://files.pythonhosted.org/packages/81/30/e935244ca6165187ae8be876b6316ae201b71485538ffac1d718843025a9/wheel-0.31.1-py2.py3-none-any.whl#sha256=80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f -homepage_url: https://bitbucket.org/pypa/wheel -license_expression: mit -licenses: - - file: mit.LICENSE - key: mit - name: MIT License - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:mit -notice_file: wheel.LICENSE -owner: Python Packaging Authority -owner_url: https://www.pypa.io/en/latest/ -vcs_repository: https://bitbucket.org/pypa/wheel -vcs_tool: hg diff --git a/thirdparty/wheel.LICENSE b/thirdparty/wheel.LICENSE deleted file mode 100644 index c3441e6c..00000000 --- a/thirdparty/wheel.LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -"wheel" copyright (c) 2012-2014 Daniel Holth and -contributors. - -The MIT License - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. diff --git a/thirdparty/wincertstore-0.2-py2.py3-none-any.whl b/thirdparty/wincertstore-0.2-py2.py3-none-any.whl deleted file mode 100644 index d3bde983..00000000 Binary files a/thirdparty/wincertstore-0.2-py2.py3-none-any.whl and /dev/null differ diff --git a/thirdparty/wincertstore.ABOUT b/thirdparty/wincertstore.ABOUT deleted file mode 100644 index 50e0dd6d..00000000 --- a/thirdparty/wincertstore.ABOUT +++ /dev/null @@ -1,15 +0,0 @@ -about_resource: wincertstore-0.2-py2.py3-none-any.whl -name: wincertstore -version: '0.2' -contact: christian@python.org -description: | - Python module to extract CA and CRL certs from Windows' cert store (ctypes based). -download_url: https://pypi.python.org/packages/2.7/w/wincertstore/wincertstore-0.2-py2.py3-none-any.whl -homepage_url: https://bitbucket.org/tiran/wincertstore -license_expression: python -licenses: - - file: python.LICENSE - key: python - name: Python Software Foundation License v2 - url: https://enterprise.dejacode.com/urn/?urn=urn:dje:license:python -owner: Christian Heimes diff --git a/thirdparty/wincertstore.LICENSE b/thirdparty/wincertstore.LICENSE deleted file mode 100644 index 311690c6..00000000 --- a/thirdparty/wincertstore.LICENSE +++ /dev/null @@ -1,49 +0,0 @@ -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python -alone or in any derivative version, provided, however, that PSF's -License Agreement and PSF's notice of copyright, i.e., "Copyright (c) -2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative -version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - diff --git a/tox.ini b/tox.ini index d7bd368f..6005c4ad 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{27,36} +envlist = py{36,37,38,39} [testenv] deps = pytest