diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..177cf82 --- /dev/null +++ b/.flake8 @@ -0,0 +1,22 @@ +[flake8] +max-line-length = 88 +enable-extensions = G +extend-ignore = + G200, G202, + # E501 line too long + E501 + # E741 ambiguous variable name + E741 + # E266 too many leading '#' for block commen + E266 + # E731 do not assign a lambda expression, use a def + E731 + # E203 whitespace before ':' + E203 + # E231 whitespace after ':' + E231 + # E221 multiple spaces before operator + E221 + # E222 multiple spaces after operator + E222 +per-file-ignores = diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..18aff79 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" + labels: + - "github-actions" + - "dependabot" diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..6dd6bd3 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,6 @@ +changelog: + exclude: + authors: + - dependabot + - pre-commit-ci + - meeseeksmachine diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1930dc3 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,69 @@ +name: Tests +on: + push: + branches: ["main"] + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Run "pre-commit run --all-files --hook-stage=manual" + pre-commit: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + - uses: pre-commit/action@v3.0.1 + with: + extra_args: --all-files --hook-stage=manual + - name: Help message if pre-commit fail + if: ${{ failure() }} + run: | + echo "You can install pre-commit hooks to automatically run formatting" + echo "on each commit with:" + echo " pre-commit install" + echo "or you can run by hand on staged files with" + echo " pre-commit run" + echo "or after-the-fact on already committed files with" + echo " pre-commit run --all-files --hook-stage=manual" + + build: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Base Setup + uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - name: Install the Python dependencies + run: | + pip install -r requirements-test.txt + - name: Run the tests + run: python -m pytest -vv -raXs || python -m pytest -vv -raXs --lf + - name: Start the App + env: + GITHUB_INTEGRATION_ID: 812 + GITHUB_BOT_NAME: meeseeksdev-test + WEBHOOK_SECRET: foo + PERSONAL_ACCOUNT_NAME: snuffy + PERSONAL_ACCOUNT_TOKEN: token + TESTING: true + run: | + set -eux + python -m meeseeksdev & + TASK_PID=$! + # Make sure the task is running + ps -p $TASK_PID + # Connect to the task + python .github/workflows/connect.py + # Kill the task + kill $TASK_PID + wait $TASK_PID diff --git a/.github/workflows/connect.py b/.github/workflows/connect.py new file mode 100644 index 0000000..392efe6 --- /dev/null +++ b/.github/workflows/connect.py @@ -0,0 +1,23 @@ +import errno +import time +from urllib import request +from urllib.error import URLError + +t0 = time.time() +found = False +url = "http://localhost:5000" + +while (time.time() - t0) < 60: + try: + request.urlopen(url) + found = True + break + except URLError as e: + if e.reason.errno == errno.ECONNREFUSED: # type:ignore + time.sleep(1) + continue + raise + + +if not found: + raise ValueError(f"Could not connect to {url}") diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000..3767677 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,23 @@ +[mypy] +check_untyped_defs = true +disallow_incomplete_defs = true +no_implicit_optional = true +pretty = true +show_error_context = true +show_error_codes = true +strict_equality = true +strict_optional = true +warn_no_return = true +warn_return_any = true +warn_unused_configs = true +warn_unused_ignores = true +warn_redundant_casts = true + +[mypy-keen] +ignore_missing_imports = True + +[mypy-there] +ignore_missing_imports = True + +[mypy-yieldbreaker] +ignore_missing_imports = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..4bfe88f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,77 @@ +default_language_version: + node: system + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: end-of-file-fixer + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: requirements-txt-fixer + - id: check-added-large-files + - id: check-case-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: forbid-new-submodules + - id: check-builtin-literals + - id: trailing-whitespace + + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + args: ["--line-length", "100"] + + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + files: \.py$ + args: [--profile=black] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.991 + hooks: + - id: mypy + args: ["--config-file", ".mypy.ini"] + additional_dependencies: [types-requests, types-PyYAML, types-mock, tornado, black, pytest, gitpython, pyjwt] + + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.10.1 + hooks: + - id: validate-pyproject + stages: [manual] + + - repo: https://github.com/executablebooks/mdformat + rev: 0.7.16 + hooks: + - id: mdformat + + - repo: https://github.com/asottile/pyupgrade + rev: v3.2.2 + hooks: + - id: pyupgrade + args: [--py38-plus] + + - repo: https://github.com/PyCQA/doc8 + rev: v1.0.0 + hooks: + - id: doc8 + args: [--max-line-length=200] + exclude: docs/source/other/full-config.rst + stages: [manual] + + - repo: https://github.com/PyCQA/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + ["flake8-bugbear==22.6.22", "flake8-implicit-str-concat==0.2.0"] + stages: [manual] + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.19.2 + hooks: + - id: check-github-workflows diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..deda0a5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,103 @@ +# Contributing + +## Test Deployment + +- Install the [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli#download-and-install). + +You will need to have an account in Heroku. + +Log in to Heroku: + +```bash +heroku login +``` + +If creating, run: + +```bash +heroku create meeseeksdev-$USER +``` + +Otherwise, run: + +```bash +heroku git:remote -a meeseeksdev-$USER +``` + +Then run: + +```bash +git push heroku $(git rev-parse --abbrev-ref HEAD):master +heroku open +``` + +To view the logs in a terminal window, use: + +```bash +heroku logs --app meeseeksdev=$USER -t +``` + +### GitHub App Configuration + +Create a GitHub App for testing on your account +Homepage URL: https://meeseeksdev-$USER.herokuapp.com/ +Webhook URL: https://meeseeksdev-$USER.herokuapp.com/webhook +Webhook Secret: Set and store as WEBHOOK_SECRET env variable +Private Key: Generate and store as B64KEY env variable + +Grant write access to content, issues, and users. +Subscribe to Issue and Issue Comment Events. + +Install the application on your user account, at least in your MeeseeksDev fork. + +### Heroku Configuration + +You will need a Github token with access to cancel builds. This + +This needs to be setup on the [Heroku Application settings](https://dashboard.heroku.com/apps/jupyterlab-bot/settings) + +On the `Config Vars`. section set the following keys:: + +``` +GITHUB_INTEGRATION_ID="" +B64KEY="" +GITHUB_BOT_NAME="" +WEBHOOK_SECRET="" +PERSONAL_ACCOUNT_NAME="" +PERSONAL_ACCOUNT_TOKEN="" +``` + +### Code Styling + +`MeeseeksDev` has adopted automatic code formatting so you shouldn't +need to worry too much about your code style. +As long as your code is valid, +the pre-commit hook should take care of how it should look. +`pre-commit` and its associated hooks will automatically be installed when +you run `pip install -e ".[test]"` + +To install `pre-commit` manually, run the following:: + +```shell +pip install pre-commit +pre-commit install +``` + +You can invoke the pre-commit hook by hand at any time with: + +```shell +pre-commit run +``` + +which should run any autoformatting on your code +and tell you about any errors it couldn't fix automatically. +You may also install [black integration](https://github.com/psf/black#editor-integration) +into your text editor to format code automatically. + +If you have already committed files before setting up the pre-commit +hook with `pre-commit install`, you can fix everything up using +`pre-commit run --all-files`. You need to make the fixing commit +yourself after that. + +Some of the hooks only run on CI by default, but you can invoke them by +running with the `--hook-stage manual` argument. diff --git a/Makefile b/Makefile index 5d68d5e..c898ad2 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: clean - + clean: find . -name '*.pyc' -delete diff --git a/Readme.md b/README.md similarity index 65% rename from Readme.md rename to README.md index 676dc63..3c31cb7 100644 --- a/Readme.md +++ b/README.md @@ -4,32 +4,32 @@ A base for stateless GitHub Bot,and one hosted implementation thereof. See what is a [Meeseeks and a MeeseeksBox](https://www.youtube.com/watch?v=qUYvIAP3qQk). +See [usage statistics](https://meeseeksbox.github.io/). + ## Hosted for you We host MeeseeksBox(es) and will expose them as GitHub Integrations so you don't have to host and run your own. You can if you want, it should be pretty -simple. +simple. The advantage of having one and only one box, is to do cross repository -operations (and fix security bugs). +operations (and fix security bugs). The drawback is if there is a security issue, then we're screwed. -## Activate on your Repo - -1) Head [there](https://github.com/apps/meeseeksdev/) and activate -MeeseeksDev on repos you have access to. +## Activate on your Repo -2) On a repository with MeeseeksDev installed say: `@MeeseeksDev Hello` to be -sure MeeseeksDev is correctly installed. +1. Head [there](https://github.com/apps/lumberbot-app/) and activate + MeeseeksDev on repos you have access to. -3) Enjoy +1. On a repository with MeeseeksDev installed say: `@MeeseeksDev Hello` to be + sure MeeseeksDev is correctly installed. -Beta Phase: During Beta phase repository/users need to be vetted/whitelisted -open an issue if you wish to participate. +1. Enjoy -You might also want to tell your CI-integration (like travis-ci) **not** to test the **push** __and__ **the merge**. +You might also want to tell your CI-integration (like travis-ci) **not** to test the **push** __and__ **the merge**. To do so use: + ``` branches: except: @@ -54,20 +54,17 @@ users: - ... ``` -This will allow `` to ask `@meeseeksdev` to perform above commands. +This will allow `` to ask `@meeseeksdev` to perform above commands. The conf file is the one that sits on the repository default branch (usually `master`). - - - ## What can a MeeseeksBox do ? Comment on a Pr or issue. -You _may_ put multiple commands, one per line. +You _may_ put multiple commands, one per line. -MrMeeseeks _may_ not like what you ask, and just ignore you. +MrMeeseeks _may_ not like what you ask, and just ignore you. ### @MeeseeksDev hello @@ -77,46 +74,70 @@ Respond with To test whether a Meeseeks understand you. -### @MeeseeksDev backport [to] {branch} +### @MeeseeksDev backport \[to\] {branch} If issued from a PR which is merged, attempt to backport (cherry-pick the merge commit) on an older branch and submit a PR with this backport (on said branch) -Apply origin-pr labels and milestone to backport. +Apply origin-pr labels and milestone to backport. -- No option to push directly (yet), if implemented should apply only with clean backport. +- No option to push directly (yet), if implemented should apply only with clean backport. - Investigate what to do in case of conflict - - likely commit with conflict, and let maintainers resolve conflict + - likely commit with conflict, and let maintainers resolve conflict Repo admins only Note: Cloning can take a long-time. So expect MrMeeseeks to be busy while this happen. Also heroku has a 2min deadline and other limitations, so MrMeeseeks can -likely be killed. I haven't implemented a queue yet. +likely be killed. I haven't implemented a queue yet. + +### @MeeseeksDev black + +If issued from a PR, will apply black to commits made in this PR and push +the updated commits. + +You can also use "blackify" as an alias. + +Repo admins only, we plan to make it available to PR authors as well. + +MeeseeksDev Bot needs to be installed on the PR source repository for this to work. +If it's not it will ask you to do so. + +### @MeeseeksDev pre-commit + +If issued from a PR, will apply pre-commit to this PR and push +a commit with the changes made. If no changes are made, or the changes +cannot be automatically fixed, it will show a comment in the PR and bail. + +You can also use "precommit" as an alias. + +Repo admins only, we plan to make it available to PR authors as well. + +MeeseeksDev Bot needs to be installed on the PR source repository for this to work. +If it's not it will ask you to do so. ### @MeeseeksDev pep8ify (in progress) If issued from a PR, will apply autopep8 to the current lines changed by this -PR, and push an extra commit to it that fixes pep8. +PR, and push an extra commit to it that fixes pep8. Code in progress and due to GitHub API limitation only works if MeeseeksDev -also available on Source repo of the PR. +also available on Source repo of the PR. -Repo admins only, plan to make it available to PR author as well. +Repo admins only, plan to make it available to PR author as well. MeeseeksDev Bot need to be installed on the PR source repository for this to work. -If it's not it will ask you to do so. +If it's not it will ask you to do so. -### @MeeseeksDev migrate [to] {target org/repo} +### @MeeseeksDev migrate \[to\] {target org/repo} Needs MeeseeksBox to be installed on both current and target repo. Command -issuer to be admin on both. +issuer to be admin on both. MeeseeksDev will open a similar issue, replicate all comments with links to -first, migrate labels (if possible). - +first, migrate labels (if possible). ### @MeeseeksDev close @@ -134,19 +155,24 @@ Tag with said tags if availlable (comma separated, need to be exact match) Remove said tags if present (comma separated, need to be exact match) -### @MeeseeksDev merge [merge|squash|rebase] +### @MeeseeksDev merge \[merge|squash|rebase\] -Issuer needs at least write permission. +Issuer needs at least write permission. If Mergeable, Merge current PR using said methods (`merge` if no arguments) +## Command Extras + +You can be polite and use "please" with any of the commands, e.g. "@Meeseeksdev please close". + +You can optionally use the word "run" in the command, e.g. "@Meeseeksdev please run pre-commit". ## Simple extension. Most extension and new command for the MeeseeksBox are only one function, for example here is how to let everyone request the zen of Python: -```python +````python from textwrap import dedent @everyone @@ -166,7 +192,7 @@ def zen(*, session, payload, arguments): ``` """ )) -``` +```` The `session` object is authenticated with the repository the command came from. If you need to authenticate with another repository with MeeseeksBox installed `yield` the `org/repo` slug. @@ -182,30 +208,21 @@ def foo(*, session, payload, argument): session.post_comment("Sorry Jerry you are not allowed to do that.") ``` - # Why do you request so much permission ? GitHub API does not allow to change permissions once given (yet). We don't want you to go though the process of reinstalling all integrations. -We would like to request less permission if necessary. - +We would like to request less permission if necessary. # Setup. -These are the environment variable that need to be set. - - - `INTEGRATION_ID` The integration ID given to you by GitHub when you create - an integration - - `BOTNAME` Name of the integration on GitHub, should be without the leading - `@`, and with the `[bot]`. This is used for the bot to react to his own name, and not reply to itself... - - TODO +See CONTIBUTING.md for for information. # Warnings This is still alpha software, user and org that can use it are still hardcoded. -If you want access open an issue for me to whitelist your org and users. +If you want access open an issue for me to allowlist your org and users. Because of GitHub API limitation, MeeseeksBox can not yet make the distinction between read-only and read-write collaborators. @@ -218,4 +235,4 @@ heroku addons:create keen ## Changelog - Oct 31st, Backport now support squash-merge +- 2017-10-31: Backport now support squash-merge diff --git a/meeseeksdev/__init__.py b/meeseeksdev/__init__.py index 5c1d5db..d8ee694 100644 --- a/meeseeksdev/__init__.py +++ b/meeseeksdev/__init__.py @@ -1,15 +1,37 @@ """ Meeseeksbox main app module """ -import os import base64 +import os import signal -org_whitelist = [ +from .commands import close, help_make, merge, migrate_issue_request +from .commands import open as _open +from .commands import ready +from .meeseeksbox.commands import ( + black_suggest, + blackify, + debug, + party, + precommit, + replyuser, + safe_backport, + say, + tag, + untag, + zen, +) +from .meeseeksbox.core import Config, MeeseeksBox + +org_allowlist = [ "MeeseeksBox", "Jupyter", "IPython", "JupyterLab", + "jupyter-server", + "jupyter-widgets", + "voila-dashboards", + "jupyter-xeus", "Carreau", "matplotlib", "scikit-learn", @@ -17,9 +39,9 @@ "scikit-image", ] -usr_blacklist = [] +usr_denylist: list = [] -usr_whitelist = [ +usr_allowlist = [ "Carreau", "gnestor", "ivanov", @@ -39,6 +61,7 @@ "lgpage", "jasongrout", "ian-r-rose", + "kevin-bates", # matplotlib people "tacaswell", "QuLogic", @@ -68,12 +91,12 @@ def load_config_from_env(): """ Load the configuration, for now stored in the environment """ - config = {} + config: dict = {} - integration_id = os.environ.get("GITHUB_INTEGRATION_ID") + integration_id_str = os.environ.get("GITHUB_INTEGRATION_ID") botname = os.environ.get("GITHUB_BOT_NAME", None) - if not integration_id: + if not integration_id_str: raise ValueError("Please set GITHUB_INTEGRATION_ID") if not botname: @@ -83,9 +106,12 @@ def load_config_from_env(): botname = botname.replace("@", "") at_botname = "@" + botname - integration_id = int(integration_id) + integration_id = int(integration_id_str) - config["key"] = base64.b64decode(bytes(os.environ.get("B64KEY"), "ASCII")) + if "B64KEY" in os.environ: + config["key"] = base64.b64decode(bytes(os.environ["B64KEY"], "ASCII")) + elif "TESTING" not in os.environ: + raise ValueError("Missing B64KEY environment variable") config["botname"] = botname config["at_botname"] = at_botname config["integration_id"] = integration_id @@ -99,36 +125,12 @@ def load_config_from_env(): # for some functionalities of mr-meeseeks. Indeed, github does not allow # cross repositories pull-requests with Applications, so I use a personal # account just for that. - config["personnal_account_name"] = os.environ.get("PERSONAL_ACCOUNT_NAME") - config["personnal_account_token"] = os.environ.get("PERSONAL_ACCOUNT_TOKEN") + config["personal_account_name"] = os.environ.get("PERSONAL_ACCOUNT_NAME") + config["personal_account_token"] = os.environ.get("PERSONAL_ACCOUNT_TOKEN") return Config(**config).validate() -from .meeseeksbox.core import MeeseeksBox -from .meeseeksbox.core import Config -from .meeseeksbox.commands import ( - replyuser, - zen, - tag, - untag, - blackify, - black_suggest, - quote, - say, - debug, - party, - safe_backport, -) -from .commands import ( - close, - open as _open, - migrate_issue_request, - ready, - merge, - help_make, -) - green = "\x1b[0;32m" yellow = "\x1b[0;33m" blue = "\x1b[0;34m" @@ -144,10 +146,13 @@ def main(): if app_v: import keen - keen.add_event("deploy", {"version": int(app_v[1:])}) - config.org_whitelist = org_whitelist + [o.lower() for o in org_whitelist] - config.user_whitelist = usr_whitelist + [u.lower() for u in usr_whitelist] - config.user_blacklist = usr_blacklist + [u.lower() for u in usr_blacklist] + try: + keen.add_event("deploy", {"version": int(app_v[1:])}) + except Exception as e: + print(e) + config.org_allowlist = org_allowlist + [o.lower() for o in org_allowlist] + config.user_allowlist = usr_allowlist + [u.lower() for u in usr_allowlist] + config.user_denylist = usr_denylist + [u.lower() for u in usr_denylist] commands = { "hello": replyuser, "zen": zen, @@ -161,7 +166,10 @@ def main(): "autopep8": blackify, "reformat": blackify, "black": blackify, + "blackify": blackify, "suggestions": black_suggest, + "pre-commit": precommit, + "precommit": precommit, "ready": ready, "merge": merge, "say": say, diff --git a/meeseeksdev/commands.py b/meeseeksdev/commands.py index d1357b3..e258926 100644 --- a/meeseeksdev/commands.py +++ b/meeseeksdev/commands.py @@ -2,11 +2,12 @@ Define a few commands """ -from .meeseeksbox.utils import Session, fix_issue_body, fix_comment_body - -from .meeseeksbox.scopes import admin, write, everyone - from textwrap import dedent +from typing import Generator, Optional + +from .meeseeksbox.commands import tag, untag +from .meeseeksbox.scopes import everyone, pr_author, write +from .meeseeksbox.utils import Session, fix_comment_body, fix_issue_body def _format_doc(function, name): @@ -15,7 +16,7 @@ def _format_doc(function, name): else: doc = function.__doc__.splitlines() first, other = doc[0], "\n".join(doc[1:]) - return "`@meeseeksdev {} {}` ({}) \n{} ".format(name, first, function.scope, other) + return f"`@meeseeksdev {name} {first}` ({function.scope}) \n{other} " def help_make(commands): @@ -50,11 +51,15 @@ def open(*, session, payload, arguments, local_config=None): @write def migrate_issue_request( - *, session: Session, payload: dict, arguments: str, local_config=None -): - """[to] {org}/{repo} - -Need to be admin on target repo. Replicate all comments on target repo and close current on. + *, + session: Session, + payload: dict, + arguments: str, + local_config: Optional[dict] = None, +) -> Generator: + """[to] {org}/{repo} + + Need to be admin on target repo. Replicate all comments on target repo and close current on. """ """Todo: @@ -75,9 +80,7 @@ def migrate_issue_request( session.post_comment( payload["issue"]["comments_url"], body="I'm afraid I can't do that. Maybe I need to be installed on target repository ?\n" - "Click [here](https://github.com/integrations/meeseeksdev/installations/new) to do that.".format( - botname="meeseeksdev" - ), + "Click [here](https://github.com/integrations/meeseeksdev/installations/new) to do that.", ) return @@ -94,16 +97,20 @@ def migrate_issue_request( if original_labels: available_labels = target_session.ghrequest( "GET", - "https://api.github.com/repos/{org}/{repo}/labels".format( - org=org, repo=repo - ), + f"https://api.github.com/repos/{org}/{repo}/labels", None, ).json() available_labels = [l["name"] for l in available_labels] - migrate_labels = [l for l in original_labels if l in available_labels] - not_set_labels = [l for l in original_labels if l not in available_labels] + migrate_labels = [ + l for l in original_labels if (l in available_labels and l != "Still Needs Manual Backport") + ] + not_set_labels = [ + l + for l in original_labels + if (l not in available_labels and l != "Still Needs Manual Backport") + ] new_response = target_session.create_issue( org, @@ -123,9 +130,7 @@ def migrate_issue_request( new_issue = new_response.json() new_comment_url = new_issue["comments_url"] - original_comments = session.ghrequest( - "GET", payload["issue"]["comments_url"], None - ).json() + original_comments = session.ghrequest("GET", payload["issue"]["comments_url"], None).json() for comment in original_comments: if comment["id"] == request_id: @@ -139,9 +144,7 @@ def migrate_issue_request( ) if not_set_labels: - body = "I was not able to apply the following label(s): %s " % ",".join( - not_set_labels - ) + body = "I was not able to apply the following label(s): %s " % ",".join(not_set_labels) target_session.post_comment(new_comment_url, body=body) session.post_comment( @@ -151,17 +154,13 @@ def migrate_issue_request( session.ghrequest("PATCH", payload["issue"]["url"], json={"state": "closed"}) -from .meeseeksbox.scopes import pr_author, write -from .meeseeksbox.commands import tag, untag - - @pr_author @write def ready(*, session, payload, arguments, local_config=None): """{no arguments} Remove "waiting for author" tag, adds "need review" tag. Can also be issued - if you are the current PR author even if you are not admin. + if you are the current PR author even if you are not admin. """ tag(session, payload, "need review") untag(session, payload, "waiting for author") @@ -184,9 +183,7 @@ def merge(*, session, payload, arguments, method="merge", local_config=None): print("== Collecting data on Pull-request...") r = session.ghrequest( "GET", - "https://api.github.com/repos/{}/{}/pulls/{}".format( - org_name, repo_name, prnumber - ), + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", json=None, ) pr_data = r.json() diff --git a/meeseeksdev/meeseeksbox/__init__.py b/meeseeksdev/meeseeksbox/__init__.py index dee3012..067b45c 100644 --- a/meeseeksdev/meeseeksbox/__init__.py +++ b/meeseeksdev/meeseeksbox/__init__.py @@ -7,41 +7,9 @@ handle authencation of user. """ -import os -import base64 -from .core import Config -from .core import MeeseeksBox +from .core import Config # noqa +from .core import MeeseeksBox # noqa version_info = (0, 0, 2) __version__ = ".".join(map(str, version_info)) - - -def load_config_from_env(): - """ - Load the configuration, for now stored in the environment - """ - config = {} - - integration_id = os.environ.get("GITHUB_INTEGRATION_ID") - botname = os.environ.get("GITHUB_BOT_NAME", None) - - if not integration_id: - raise ValueError("Please set GITHUB_INTEGRATION_ID") - - if not botname: - raise ValueError("Need to set a botname") - if "@" in botname: - print("Don't include @ in the botname !") - - botname = botname.replace("@", "") - at_botname = "@" + botname - integration_id = int(integration_id) - - config["key"] = base64.b64decode(bytes(os.environ.get("B64KEY"), "ASCII")) - config["botname"] = botname - config["at_botname"] = at_botname - config["integration_id"] = integration_id - config["webhook_secret"] = os.environ.get("WEBHOOK_SECRET") - - return Config(**config).validate() diff --git a/meeseeksdev/meeseeksbox/commands.py b/meeseeksdev/meeseeksbox/commands.py index 679f523..9c06321 100644 --- a/meeseeksdev/meeseeksbox/commands.py +++ b/meeseeksdev/meeseeksbox/commands.py @@ -2,25 +2,22 @@ Define a few commands """ -import random import os -import re -import subprocess -import git import pipes -import mock -import keen +import random +import re +import sys import time import traceback - -import sys +from pathlib import Path from textwrap import dedent +from typing import Generator, Optional +from unittest import mock -# from friendlyautopep8 import run_on_cwd - -from .utils import Session, fix_issue_body, fix_comment_body +import git from .scopes import admin, everyone, write +from .utils import Session, add_event, fix_comment_body, fix_issue_body, run green = "\033[0;32m" yellow = "\033[0;33m" @@ -106,53 +103,54 @@ def zen(*, session, payload, arguments, local_config=None): def replyadmin(*, session, payload, arguments, local_config=None): comment_url = payload["issue"]["comments_url"] user = payload["issue"]["user"]["login"] - session.post_comment( - comment_url, "Hello @{user}. Waiting for your orders.".format(user=user) - ) + session.post_comment(comment_url, f"Hello @{user}. Waiting for your orders.") -def _compute_pwd_changes(whitelist): - import black - from difflib import SequenceMatcher - from pathlib import Path +def _compute_pwd_changes(allowlist): import glob + from difflib import SequenceMatcher + + import black + post_changes = [] import os - print('== pwd', os.getcwd()) - print('== listdir', os.listdir()) - for p in glob.glob('**/*.py', recursive=True): - print('=== scanning', p, p in whitelist) - if p not in whitelist: + print("== pwd", os.getcwd()) + print("== listdir", os.listdir()) + + for path in glob.glob("**/*.py", recursive=True): + print("=== scanning", path, path in allowlist) + if path not in allowlist: # we don't touch files not in this PR. continue - p = Path(p) + p = Path(path) old = p.read_text() new = black.format_str(old, mode=black.FileMode()) - if new != old: - print('will differ') + if new != old: + print("will differ") nl = new.splitlines() ol = old.splitlines() s = SequenceMatcher(None, ol, nl) for t, a1, a2, b1, b2 in s.get_opcodes(): - if t == 'replace': + if t == "replace": + + c = "```suggestion\n" - c = '```suggestion\n' - for n in nl[b1:b2]: - c+=n - c+='\n' - c+='```' + c += n + c += "\n" + c += "```" ch = (p.as_posix(), a1, a2, c) post_changes.append(ch) return post_changes + @admin def black_suggest(*, session, payload, arguments, local_config=None): print("===== reformatting suggestions. =====") prnumber = payload["issue"]["number"] - prtitle = payload["issue"]["title"] + # prtitle = payload["issue"]["title"] org_name = payload["repository"]["owner"]["login"] repo_name = payload["repository"]["name"] @@ -160,30 +158,25 @@ def black_suggest(*, session, payload, arguments, local_config=None): print("== Collecting data on Pull-request...") r = session.ghrequest( "GET", - "https://api.github.com/repos/{}/{}/pulls/{}".format( - org_name, repo_name, prnumber - ), + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", json=None, ) pr_data = r.json() head_sha = pr_data["head"]["sha"] - base_sha = pr_data["base"]["sha"] - branch = pr_data["head"]["ref"] - author_login = pr_data["head"]["repo"]["owner"]["login"] + # base_sha = pr_data["base"]["sha"] + # branch = pr_data["head"]["ref"] + # author_login = pr_data["head"]["repo"]["owner"]["login"] repo_name = pr_data["head"]["repo"]["name"] - commits_url = pr_data['commits_url'] - - commits_data = session.ghrequest( "GET",commits_url).json() - + # commits_url = pr_data["commits_url"] - + # commits_data = session.ghrequest("GET", commits_url).json() # that will likely fail, as if PR, we need to bypass the fact that the # requester has technically no access to committer repo. # TODO, check if maintainer ## target_session = yield "{}/{}".format(author_login, repo_name) - ## if target_session: + ## if target_session: ## print('installed on target repository') ## atk = target_session.token() ## else: @@ -192,16 +185,16 @@ def black_suggest(*, session, payload, arguments, local_config=None): ## comment_url = payload["issue"]["comments_url"] ## session.post_comment( ## comment_url, - ## body="Would you mind installing me on your fork so that I can update your branch ? \n" + ## body="Would you mind installing me on your fork so that I can update your branch? \n" ## "Click [here](https://github.com/apps/meeseeksdev/installations/new)" - ## "to do that, and follow the instruction to add your fork." + ## "to do that, and follow the instructions to add your fork." ## "I'm going to try to push as a maintainer but this may not work." ## ) # if not target_session: # comment_url = payload["issue"]["comments_url"] # session.post_comment( # comment_url, - # body="I'm afraid I can't do that. Maybe I need to be installed on target repository ?\n" + # body="I'm afraid I can't do that. Maybe I need to be installed on target repository?\n" # "Click [here](https://github.com/apps/meeseeksdev/installations/new) to do that.".format( # botname="meeseeksdev" # ), @@ -212,17 +205,20 @@ def black_suggest(*, session, payload, arguments, local_config=None): # this process can take some time, regen token # paginated by 30 files, let's nto go that far (yet) - files_response = session.ghrequest("GET", f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/files") - pr_files = [r['filename'] for r in files_response.json()] - print('== PR contains', len(pr_files), 'files') + files_response = session.ghrequest( + "GET", + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/files", + ) + pr_files = [r["filename"] for r in files_response.json()] + print("== PR contains", len(pr_files), "files") if os.path.exists(repo_name): - print("== Cleaning up previsous work... ") - subprocess.run("rm -rf {}".format(repo_name).split(" ")) + print("== Cleaning up previous work ... ") + run(f"rm -rf {repo_name}") print("== Done cleaning ") - print(f"== Cloning repository from {org_name}/{repo_name}, this can take some time..") - process = subprocess.run( + print(f"== Cloning repository from {org_name}/{repo_name}, this can take some time ...") + process = run( [ "git", "clone", @@ -234,59 +230,57 @@ def black_suggest(*, session, payload, arguments, local_config=None): print("== Cloned..") process.check_returncode() - subprocess.run("git config --global user.email meeseeksmachine@gmail.com".split(" ")) - subprocess.run("git config --global user.name FriendlyBot".split(" ")) + run("git config --global user.email meeseeksmachine@gmail.com") + run("git config --global user.name FriendlyBot") # do the pep8ify on local filesystem repo = git.Repo(repo_name) - #branch = master - #print(f"== Fetching branch `{branch}` ...") - #repo.remotes.origin.fetch("{}:workbranch".format(branch)) - #repo.git.checkout("workbranch") - print("== Fetching Commits to reformat...") - repo.remotes.origin.fetch("{head_sha}".format(head_sha=head_sha)) - print("== All has been fetched correctly") + # branch = master + # print(f"== Fetching branch `{branch}` ...") + # repo.remotes.origin.fetch("{}:workbranch".format(branch)) + # repo.git.checkout("workbranch") + print("== Fetching Commits to reformat ...") + repo.remotes.origin.fetch(f"{head_sha}") + print("== All have been fetched correctly") repo.git.checkout(head_sha) print(f"== checked PR head {head_sha}") - print("== Computing changes....") os.chdir(repo_name) changes = _compute_pwd_changes(pr_files) - os.chdir('..') + os.chdir("..") print("... computed", len(changes), changes) - - COMFORT_FADE = 'application/vnd.github.comfort-fade-preview+json' + COMFORT_FADE = "application/vnd.github.comfort-fade-preview+json" # comment_url = payload["issue"]["comments_url"] # session.post_comment( # comment_url, # body=dedent(""" # I've rebased this Pull Request, applied `black` on all the # individual commits, and pushed. You may have trouble pushing further - # commits, but feel free to force push and ask me to reformat again. + # commits, but feel free to force push and ask me to reformat again. # """) # ) for path, start, end, body in changes: - print(f'== will suggest the following on {path} {start+1} to {end}\n', body) - if start+1 != end: + print(f"== will suggest the following on {path} {start+1} to {end}\n", body) + if start + 1 != end: data = { - "body": body, - "commit_id": head_sha, - "path": path, - "start_line": start+1, - "start_side": "RIGHT", - "line": end, - "side": "RIGHT" + "body": body, + "commit_id": head_sha, + "path": path, + "start_line": start + 1, + "start_side": "RIGHT", + "line": end, + "side": "RIGHT", } try: - resp = session.ghrequest( - "POST", - f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/comments", - json=data, - override_accept_header=COMFORT_FADE, + _ = session.ghrequest( + "POST", + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/comments", + json=data, + override_accept_header=COMFORT_FADE, ) except Exception: # likely unprecessable entity out of range @@ -294,166 +288,325 @@ def black_suggest(*, session, payload, arguments, local_config=None): else: # we can't seem to do single line with COMFORT_FADE data = { - "body": body, - "commit_id": head_sha, - "path": path, - "line": end, - "side": "RIGHT" + "body": body, + "commit_id": head_sha, + "path": path, + "line": end, + "side": "RIGHT", } try: - resp = session.ghrequest( - "POST", - f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/comments", - json=data, + _ = session.ghrequest( + "POST", + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}/comments", + json=data, ) except Exception: - # likely unprecessable entity out of range + # likely unprocessable entity out of range pass if os.path.exists(repo_name): print("== Cleaning up repo... ") - subprocess.run("rm -rf {}".format(repo_name).split(" ")) + run(f"rm -rf {repo_name}") print("== Done cleaning ") - -@admin -def blackify(*, session, payload, arguments, local_config=None): - print("===== reformatting =====") +def prep_for_command( + name: str, + session: Session, + payload: dict, + arguments: str, + local_config: Optional[dict] = None, +) -> Generator: + """Prepare to run a command against a local checkout of a repo.""" + print(f"===== running command {name} =====") print("===== ============ =====") # collect initial payload + # https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#issue_comment prnumber = payload["issue"]["number"] - prtitle = payload["issue"]["title"] org_name = payload["repository"]["owner"]["login"] repo_name = payload["repository"]["name"] + comment_url = payload["issue"]["comments_url"] # collect extended payload on the PR + # https://docs.github.com/en/rest/reference/pulls#get-a-pull-request print("== Collecting data on Pull-request...") r = session.ghrequest( "GET", - "https://api.github.com/repos/{}/{}/pulls/{}".format( - org_name, repo_name, prnumber - ), + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", json=None, ) pr_data = r.json() head_sha = pr_data["head"]["sha"] - base_sha = pr_data["base"]["sha"] branch = pr_data["head"]["ref"] author_login = pr_data["head"]["repo"]["owner"]["login"] repo_name = pr_data["head"]["repo"]["name"] + maintainer_can_modify = pr_data["maintainer_can_modify"] - commits_url = pr_data['commits_url'] + # Check to see if we can successfully push changees to the PR. + target_session = yield f"{author_login}/{repo_name}" - commits_data = session.ghrequest( "GET",commits_url).json() - - for commit in commits_data: - if len(commit['parents']) != 1: - comment_url = payload["issue"]["comments_url"] - session.post_comment( - comment_url, - body="It looks like the history is not linear in this pull-request. I'm afraid I can't rebase.\n" - ) - return - - # so far we assume that the commit we rebase on is the first. - to_rebase_on = commits_data[0]['parents'][0]['sha'] - - # that will likely fail, as if PR, we need to bypass the fact that the - # requester has technically no access to committer repo. - # TODO, check if maintainer - target_session = yield "{}/{}".format(author_login, repo_name) - if target_session: - print('installed on target repository') + if target_session: + print("installed on target repository") atk = target_session.token() + session.post_comment(comment_url, body=f"Running {name} on this Pull Request...") else: - print('use allow edit as maintainer') + print("use allow edit as maintainer") + if maintainer_can_modify: + msg = "For now I will push as a maintainer since it is enabled." + else: + msg = 'I would push as a maintainer but I cannot unless "Allow edits from maintainers" is enabled for this Pull Request.' atk = session.token() - comment_url = payload["issue"]["comments_url"] session.post_comment( comment_url, - body="Would you mind installing me on your fork so that I can update your branch ? \n" - "Click [here](https://github.com/apps/meeseeksdev/installations/new)" - "to do that, and follow the instruction to add your fork." - "I'm going to try to push as a maintainer but this may not work." + body=f"@{author_login}, would you mind installing me on your fork so that I can update your branch?\n" + "Click [here](https://github.com/apps/meeseeksdev/installations/new) " + "to do that, and follow the instructions to add your fork.\n\n" + f"{msg}", ) - # if not target_session: - # comment_url = payload["issue"]["comments_url"] - # session.post_comment( - # comment_url, - # body="I'm afraid I can't do that. Maybe I need to be installed on target repository ?\n" - # "Click [here](https://github.com/apps/meeseeksdev/installations/new) to do that.".format( - # botname="meeseeksdev" - # ), - # ) - # return - - # clone locally - # this process can take some time, regen token + if not maintainer_can_modify: + print("=== Bailing since we do not have permissions") + return if os.path.exists(repo_name): - print("== Cleaning up previsous work... ") - subprocess.run("rm -rf {}".format(repo_name).split(" ")) + print("== Cleaning up previous work ... ") + run(f"rm -rf {repo_name}", check=True) print("== Done cleaning ") - print(f"== Cloning repository from {author_login}/{repo_name}, this can take some time..") - process = subprocess.run( + print(f"== Cloning repository from {author_login}/{repo_name}, this can take some time ...") + process = run( [ "git", "clone", - "https://x-access-token:{}@github.com/{}/{}".format( - atk, author_login, repo_name - ), + f"https://x-access-token:{atk}@github.com/{author_login}/{repo_name}", ] ) print("== Cloned..") process.check_returncode() - subprocess.run("git config --global user.email meeseeksmachine@gmail.com".split(" ")) - subprocess.run("git config --global user.name FriendlyBot".split(" ")) + run("git config --global user.email meeseeksmachine@gmail.com") + run("git config --global user.name FriendlyBot") - # do the pep8ify on local filesystem + # do the command on local filesystem repo = git.Repo(repo_name) - print(f"== Fetching branch `{branch}` to pep8ify on ...") - repo.remotes.origin.fetch("{}:workbranch".format(branch)) + print(f"== Fetching branch `{branch}` to run {name} on ...") + repo.remotes.origin.fetch(f"{branch}:workbranch") repo.git.checkout("workbranch") - print("== Fetching Commits to pep8ify...") - repo.remotes.origin.fetch("{head_sha}".format(head_sha=head_sha)) - print("== All has been fetched correctly") + print(f"== Fetching Commits to run {name} on ...") + repo.remotes.origin.fetch(f"{head_sha}") + print("== All have been fetched correctly") os.chdir(repo_name) - def lpr(*args): - print('Should run:', *args) - lpr('git rebase -x "black --fast . && git commit -a --amend --no-edit" --strategy-option=theirs --autosquash', to_rebase_on ) +def push_the_work(session, payload, arguments, local_config=None): + """Push the work down in a local repo to the remote repo.""" + prnumber = payload["issue"]["number"] + org_name = payload["repository"]["owner"]["login"] + repo_name = payload["repository"]["name"] - ## todo check error code. - subprocess.run(['git','rebase', '-x','black --fast . && git commit -a --amend --no-edit','--strategy-option=theirs','--autosquash', to_rebase_on]) + # collect extended payload on the PR + print("== Collecting data on Pull-request...") + r = session.ghrequest( + "GET", + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", + json=None, + ) + pr_data = r.json() + branch = pr_data["head"]["ref"] + repo_name = pr_data["head"]["repo"]["name"] - ## write the commit message - #msg = "Autofix pep 8 of #%i: %s" % (prnumber, prtitle) + "\n\n" - #repo.git.commit("-am", msg) + # Open the repo in the cwd + repo = git.Repo(".") - ## Push the pep8ify work + # Push the work print("== Pushing work....:") - lpr(f"pushing with workbranch:{branch}") - repo.remotes.origin.push("workbranch:{}".format(branch), force=True) - repo.git.checkout("master") + print(f"pushing with workbranch:{branch}") + succeeded = True + try: + repo.remotes.origin.push(f"workbranch:{branch}", force=True).raise_if_error() + except Exception: + succeeded = False + + # Clean up + default_branch = session.ghrequest( + "GET", f"https://api.github.com/repos/{org_name}/{repo_name}" + ).json()["default_branch"] + repo.git.checkout(default_branch) repo.branches.workbranch.delete(repo, "workbranch", force=True) + return succeeded +@admin +def precommit( + *, + session: Session, + payload: dict, + arguments: str, + local_config: Optional[dict] = None, +) -> Generator: comment_url = payload["issue"]["comments_url"] - session.post_comment( - comment_url, - body=dedent(""" - I've rebased this Pull Request, applied `black` on all the - individual commits, and pushed. You may have trouble pushing further - commits, but feel free to force push and ask me to reformat again. - """) + + """Run pre-commit against a PR and push the changes.""" + yield from prep_for_command("precommit", session, payload, arguments, local_config=local_config) + + # Make sure there is a pre-commit file. + config = Path("./.pre-commit-config.yaml") + if not config.exists(): + # Alert the caller and bail. + session.post_comment( + comment_url, + body=dedent( + """ + I was unable to fix pre-commit errors because there is no + ".pre-commit-config.yaml" file. + """ + ), + ) + return + + # Install the package in editable mode if there are local hooks. + if "repo: local" in config.read_text(): + run("pip install --user -e .") + + cmd = "pre-commit run --all-files --hook-stage=manual" + + # Run the command + process = run(cmd) + + # See if the pre-commit succeeded, meaning there was nothing to do + if process.returncode == 0: + + # Alert the caller and bail. + session.post_comment( + comment_url, + body=dedent( + """ + I was unable to run "pre-commit" because there was nothing to do. + """ + ), + ) + return + + # Add any changed files. + process = run('git commit -a -m "Apply pre-commit"') + made_changes = process.returncode == 0 + + # Run again to see if we've auto-fixed + process = run(cmd) + + # If second run fails, then we can't auto-fix + if process.returncode != 0: + + if not made_changes: + # Alert the caller and bail. + session.post_comment( + comment_url, + body=dedent( + """ + I was unable to fix pre-commit errors automatically. + Try running `pre-commit run --all-files` locally. + """ + ), + ) + return + + session.post_comment( + comment_url, + body=dedent( + """ + I was unable to fix all of the pre-commit errors automatically. Try running `pre-commit run --all-files` locally. + """ + ), + ) + + succeeded = push_the_work(session, payload, arguments, local_config=local_config) + + # Tell the caller we've finished + comment_url = payload["issue"]["comments_url"] + if succeeded: + session.post_comment( + comment_url, + body=dedent( + """ + I've applied "pre-commit" and pushed. You may have trouble pushing further + commits, but feel free to force push and ask me to run again. + """ + ), + ) + else: + session.post_comment(comment_url, body="I was unable to push due to errors") + + +@admin +def blackify(*, session, payload, arguments, local_config=None): + """Run black against all commits of on a PR and push the new commits.""" + yield from prep_for_command("blackify", session, payload, arguments, local_config=local_config) + + comment_url = payload["issue"]["comments_url"] + + prnumber = payload["issue"]["number"] + org_name = payload["repository"]["owner"]["login"] + repo_name = payload["repository"]["name"] + + r = session.ghrequest( + "GET", + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", + json=None, ) - #os.chdir("..") + pr_data = r.json() + commits_url = pr_data["commits_url"] + commits_data = session.ghrequest("GET", commits_url).json() + + for commit in commits_data: + if len(commit["parents"]) != 1: + session.post_comment( + comment_url, + body="It looks like the history is not linear in this pull-request. I'm afraid I can't rebase.\n", + ) + return + + # so far we assume that the commit we rebase on is the first. + to_rebase_on = commits_data[0]["parents"][0]["sha"] + + process = run( + [ + "git", + "rebase", + "-x", + "black --fast . && git commit -a --amend --no-edit", + "--strategy-option=theirs", + "--autosquash", + to_rebase_on, + ] + ) + + if process.returncode != 0: + session.post_comment( + comment_url, + body=dedent( + """ + I was unable to run "blackify" due to an error. + """ + ), + ) + return + + succeeded = push_the_work(session, payload, arguments, local_config=local_config) + # Tell the caller we've finished + if succeeded: + session.post_comment( + comment_url, + body=dedent( + """ + I've rebased this Pull Request, applied `black` on all the + individual commits, and pushed. You may have trouble pushing further + commits, but feel free to force push and ask me to reformat again. + """ + ), + ) + else: + session.post_comment(comment_url, body="I was unable to push due to errors") @write @@ -463,12 +616,13 @@ def safe_backport(session, payload, arguments, local_config=None): print = lambda *args, **kwargs: builtins.print(" [backport]", *args, **kwargs) - s_clone_time = 0 + s_clone_time = 0.0 s_success = False s_reason = "unknown" - s_fork_time = 0 - s_clean_time = 0 - s_ff_time = 0 + s_fork_time = 0.0 + s_clean_time = 0.0 + s_ff_time = 0.0 + s_slug = "" def keen_stats(): nonlocal s_slug @@ -478,7 +632,7 @@ def keen_stats(): nonlocal s_fork_time nonlocal s_clean_time nonlocal s_ff_time - keen.add_event( + add_event( "backport_stats", { "slug": s_slug, @@ -511,10 +665,16 @@ def keen_stats(): maybe_wrong_named_branch = False s_slug = f"{org_name}/{repo_name}" try: + default_branch = session.ghrequest( + "GET", f"https://api.github.com/repos/{org_name}/{repo_name}" + ).json()["default_branch"] existing_branches = session.ghrequest( "GET", f"https://api.github.com/repos/{org_name}/{repo_name}/branches" ).json() existing_branches_names = {b["name"] for b in existing_branches} + if target_branch not in existing_branches_names and target_branch.endswith("."): + target_branch = target_branch[:-1] + if target_branch not in existing_branches_names: print( red @@ -532,17 +692,14 @@ def keen_stats(): try: # collect extended payload on the PR - print("== Collecting data on Pull-request...") + print("== Collecting data on Pull-request ...") r = session.ghrequest( "GET", - "https://api.github.com/repos/{}/{}/pulls/{}".format( - org_name, repo_name, prnumber - ), + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls/{prnumber}", json=None, ) pr_data = r.json() merge_sha = pr_data["merge_commit_sha"] - body = pr_data["body"] milestone = pr_data["milestone"] if milestone: milestone_number = pr_data["milestone"].get("number", None) @@ -557,17 +714,23 @@ def keen_stats(): parts = milestone_title.split(".") parts[-1] = "x" infered_target_branch = ".".join(parts) - print("inferring branch....", infered_target_branch) + print("inferring branch ...", infered_target_branch) target_branch = infered_target_branch - keen.add_event("backport_infering_branch", {"infering_remove_x": 1}) + add_event("backport_infering_branch", {"infering_remove_x": 1}) if milestone_number: milestone_number = int(milestone_number) labels_names = [] try: - label_names = [l["name"] for l in pr_data["labels"]] - if not label_names and ("issue" in payload.keys()): - labels_names = [l["name"] for l in payload["issue"]["labels"]] + labels_names = [ + l["name"] for l in pr_data["labels"] if l["name"] != "Still Needs Manual Backport" + ] + if not labels_names and ("issue" in payload.keys()): + labels_names = [ + l["name"] + for l in payload["issue"]["labels"] + if l["name"] != "Still Needs Manual Backport" + ] except KeyError: print("Did not find labels|", pr_data) # clone locally @@ -583,7 +746,7 @@ def keen_stats(): for i in range(5): ff = session.personal_request("GET", frk["url"], raise_for_status=False) if ff.status_code == 200: - keen.add_event("fork_wait", {"n": i}) + add_event("fork_wait", {"n": i}) break time.sleep(1) s_fork_time = time.time() - fork_epoch @@ -593,8 +756,7 @@ def keen_stats(): if os.path.exists(repo_name): try: re_fetch_epoch = time.time() - print("FF: Git set-url origin") - subprocess.run( + run( [ "git", "remote", @@ -606,25 +768,24 @@ def keen_stats(): ).check_returncode() repo = git.Repo(repo_name) - print("FF: Git fetch master") - repo.remotes.origin.fetch("master") - repo.git.checkout("master") - print("FF: Reset hard origin/master") - subprocess.run( - ["git", "reset", "--hard", "origin/master"], cwd=repo_name + print(f"FF: Git fetch {default_branch}") + repo.remotes.origin.fetch(default_branch) + repo.git.checkout(default_branch) + run( + ["git", "reset", "--hard", f"origin/{default_branch}"], + cwd=repo_name, ).check_returncode() - print("FF: Git describe tags....") - subprocess.run(["git", "describe", "--tag"], cwd=repo_name) + run(["git", "describe", "--tag"], cwd=repo_name) re_fetch_delta = time.time() - re_fetch_epoch print(blue + f"FF took {re_fetch_delta}s") s_ff_time = re_fetch_delta - except Exception as e: + except Exception: # something went wrong. Kill repository it's going to be # recloned. clean_epoch = time.time() if os.path.exists(repo_name): - print("== Cleaning up previsous work... ") - subprocess.run("rm -rf {}".format(repo_name).split(" ")) + print("== Cleaning up previous work... ") + run(f"rm -rf {repo_name}") print("== Done cleaning ") s_clean_time = time.time() - clean_epoch import traceback @@ -632,19 +793,16 @@ def keen_stats(): traceback.print_exc() ## end optimise-fetch-experiment - clone_epoch = time.time() action = "set-url" what_was_done = "Fast-Forwarded" if not os.path.exists(repo_name): print("== Cloning current repository, this can take some time..") - process = subprocess.run( + process = run( [ "git", "clone", - "https://x-access-token:{}@github.com/{}/{}".format( - atk, org_name, repo_name - ), + f"https://x-access-token:{atk}@github.com/{org_name}/{repo_name}", ] ) process.check_returncode() @@ -653,40 +811,33 @@ def keen_stats(): s_clone_time = time.time() - clone_epoch - process = subprocess.run( + process = run( [ "git", "remote", action, - session.personnal_account_name, - f"https://x-access-token:{session.personnal_account_token}@github.com/{session.personnal_account_name}/{repo_name}", + session.personal_account_name, + f"https://x-access-token:{session.personal_account_token}@github.com/{session.personal_account_name}/{repo_name}", ], cwd=repo_name, ) print("==", what_was_done) process.check_returncode() - subprocess.run( - "git config --global user.email meeseeksmachine@gmail.com".split(" ") - ) - subprocess.run("git config --global user.name MeeseeksDev[bot]".split(" ")) + run("git config --global user.email meeseeksmachine@gmail.com") + run("git config --global user.name MeeseeksDev[bot]") # do the backport on local filesystem repo = git.Repo(repo_name) - print("== Fetching branch to backport on ... {}".format(target_branch)) - repo.remotes.origin.fetch("refs/heads/{}:workbranch".format(target_branch)) + print(f"== Fetching branch to backport on ... {target_branch}") + repo.remotes.origin.fetch(f"refs/heads/{target_branch}:workbranch") repo.git.checkout("workbranch") - print( - "== Fetching Commits to {mergesha} backport...".format(mergesha=merge_sha) - ) - repo.remotes.origin.fetch("{mergesha}".format(num=prnumber, mergesha=merge_sha)) + print(f"== Fetching Commits to {merge_sha} backport...") + repo.remotes.origin.fetch(f"{merge_sha}") print("== All has been fetched correctly") - # remove mentions from description, to avoid pings: - description = body.replace("@", " ").replace("#", " ") - print("Cherry-picking %s" % merge_sha) - args = ("-m", "1", merge_sha) + args: tuple = ("-x", "-m", "1", merge_sha) msg = "Backport PR #%i: %s" % (prnumber, prtitle) remote_submit_branch = f"auto-backport-of-pr-{prnumber}-on-{target_branch}" @@ -697,17 +848,15 @@ def keen_stats(): repo.git.cherry_pick(*args) except git.GitCommandError as e: if "is not a merge." in e.stderr: - print( - "Likely not a merge PR...Attempting squash and merge picking." - ) + print("Likely not a merge PR...Attempting squash and merge picking.") args = (merge_sha,) repo.git.cherry_pick(*args) else: raise except git.GitCommandError as e: - if ("git commit --allow-empty" in e.stderr) or ( - "git commit --allow-empty" in e.stdout + if ("git commit --allow-empty" in e.stderr.lower()) or ( + "git commit --allow-empty" in e.stdout.lower() ): session.post_comment( comment_url, @@ -721,7 +870,7 @@ def keen_stats(): s_reason = "empty commit" keen_stats() return - elif "after resolving the conflicts" in e.stderr: + elif "after resolving the conflicts" in e.stderr.lower(): # TODO, here we should also do a git merge --abort # to avoid thrashing the cache at next backport request. cmd = " ".join(pipes.quote(arg) for arg in sys.argv) @@ -739,22 +888,22 @@ def keen_stats(): 1. Checkout backport branch and update it. ``` -$ git checkout {target_branch} -$ git pull +git checkout {target_branch} +git pull ``` 2. Cherry pick the first parent branch of the this PR on top of the older branch: ``` -$ git cherry-pick -m1 {merge_sha} +git cherry-pick -x -m1 {merge_sha} ``` 3. You will likely have some merge/cherry-pick conflict here, fix them and commit: ``` -$ git commit -am {msg!r} +git commit -am {msg!r} ``` -4. Push to a named branch : +4. Push to a named branch: ``` git push YOURFORK {target_branch}:{remote_submit_branch} @@ -762,13 +911,15 @@ def keen_stats(): 5. Create a PR against branch {target_branch}, I would have named this PR: -> "Backport PR #{prnumber} on branch {target_branch}" +> "Backport PR #{prnumber} on branch {target_branch} ({prtitle})" And apply the correct labels and milestones. -Congratulation you did some good work ! Hopefully your backport PR will be tested by the continuous integration and merged soon! +Congratulations — you did some good work! Hopefully your backport PR will be tested by the continuous integration and merged soon! -If these instruction are inaccurate, feel free to [suggest an improvement](https://github.com/MeeseeksBox/MeeseeksDev). +Remember to remove the `Still Needs Manual Backport` label once the PR gets merged. + +If these instructions are inaccurate, feel free to [suggest an improvement](https://github.com/MeeseeksBox/MeeseeksDev). """, ) org = payload["repository"]["owner"]["login"] @@ -776,9 +927,7 @@ def keen_stats(): num = payload.get("issue", payload).get("number") url = f"https://api.github.com/repos/{org}/{repo}/issues/{num}/labels" print("trying to apply still needs manual backport") - reply = session.ghrequest( - "POST", url, json=["Still Needs Manual Backport"] - ) + reply = session.ghrequest("POST", url, json=["Still Needs Manual Backport"]) print("Should be applied:", reply) s_reason = "conflicts" keen_stats() @@ -786,7 +935,7 @@ def keen_stats(): else: session.post_comment( comment_url, - "Oops, something went wrong applying the patch... Please have a look at my logs.", + "Oops, something went wrong applying the patch ... Please have a look at my logs.", ) print(e.stderr) print("----") @@ -799,9 +948,13 @@ def keen_stats(): session.post_comment( comment_url, "Hum, I actually crashed, that should not have happened." ) - print("\n" + e.stderr.decode("utf8", "replace"), file=sys.stderr) + if hasattr(e, "stderr"): + print( + "\n" + e.stderr.decode("utf8", "replace"), + file=sys.stderr, + ) print("\n" + repo.git.status(), file=sys.stderr) - keen.add_event("error", {"git_crash": 1}) + add_event("error", {"git_crash": 1}) s_reason = "Unknown error line 501" keen_stats() @@ -817,26 +970,36 @@ def keen_stats(): # Push the backported work print("== Pushing work....:") + succeeded = True try: - print( - f"Tryign to push to {remote_submit_branch} of {session.personnal_account_name}" - ) - repo.remotes[session.personnal_account_name].push( - "workbranch:{}".format(remote_submit_branch) - ) + print(f"Trying to push to {remote_submit_branch} of {session.personal_account_name}") + repo.remotes[session.personal_account_name].push( + f"workbranch:{remote_submit_branch}" + ).raise_if_error() except Exception as e: import traceback - traceback.print_exc() + content = traceback.format_exc() + print(content.replace(session.personal_account_token, "...")) + print("could not push to self remote") s_reason = "Could not push" keen_stats() - # TODO comment on issue + + session.post_comment( + comment_url, + f"Could not push to {remote_submit_branch} due to error, aborting.", + ) print(e) - repo.git.checkout("master") + succeeded = False + + repo.git.checkout(default_branch) repo.branches.workbranch.delete(repo, "workbranch", force=True) - # TODO checkout master and get rid of branch + # TODO checkout the default_branch and get rid of branch + + if not succeeded: + return # Make the PR on GitHub print( @@ -847,13 +1010,11 @@ def keen_stats(): ) new_pr = session.personal_request( "POST", - "https://api.github.com/repos/{}/{}/pulls".format(org_name, repo_name), + f"https://api.github.com/repos/{org_name}/{repo_name}/pulls", json={ "title": f"Backport PR #{prnumber} on branch {target_branch} ({prtitle})", "body": msg, - "head": "{}:{}".format( - session.personnal_account_name, remote_submit_branch - ), + "head": f"{session.personal_account_name}:{remote_submit_branch}", "base": target_branch, }, ).json() @@ -861,24 +1022,24 @@ def keen_stats(): new_number = new_pr["number"] resp = session.ghrequest( "PATCH", - "https://api.github.com/repos/{}/{}/issues/{}".format( - org_name, repo_name, new_number - ), + f"https://api.github.com/repos/{org_name}/{repo_name}/issues/{new_number}", json={"milestone": milestone_number, "labels": labels_names}, ) # print(resp.json()) except Exception as e: extra_info = "" if maybe_wrong_named_branch: - extra_info = "\n\n It seem that the branch you are trying to backport to does not exists." + extra_info = ( + "\n\n It seems that the branch you are trying to backport to does not exist." + ) session.post_comment( comment_url, - "Something went wrong ... Please have a look at my logs." + extra_info, + "Something went wrong ... Please have a look at my logs." + extra_info, ) - keen.add_event("error", {"unknown_crash": 1}) + add_event("error", {"unknown_crash": 1}) print("Something went wrong") print(e) - s_reason = "Remote branches does not exists" + s_reason = "Remote branch does not exist" keen_stats() raise @@ -900,11 +1061,11 @@ def tag(session, payload, arguments, local_config=None): num = payload.get("issue", payload.get("pull_request")).get("number") url = f"https://api.github.com/repos/{org}/{repo}/issues/{num}/labels" arguments = arguments.replace("'", '"') - quoted = re.findall(r'\"(.+?)\"',arguments.replace("'", '"')) + quoted = re.findall(r"\"(.+?)\"", arguments.replace("'", '"')) for q in quoted: - arguments = arguments.replace('"%s"' % q, '') + arguments = arguments.replace('"%s"' % q, "") tags = [arg.strip() for arg in arguments.split(",") if arg.strip()] + quoted - print('raw tags:', tags) + print("raw tags:", tags) to_apply = [] not_applied = [] try: @@ -915,51 +1076,48 @@ def tag(session, payload, arguments, local_config=None): label_payloads = [label_payload] def get_next_link(req): - all_links = req.headers.get('Link') + all_links = req.headers.get("Link") if 'rel="next"' in all_links: - links = all_links.split(',') - next_link = [l for l in links if 'next' in l][0] # assume only one. + links = all_links.split(",") + next_link = [l for l in links if "next" in l][0] # assume only one. if next_link: - return next_link.split(';')[0].strip(' <>') - + return next_link.split(";")[0].strip(" <>") # let's assume no more than 200 labels resp = label_payload try: for i in range(10): - print('get labels page',i) + print("get labels page", i) next_link = get_next_link(resp) if next_link: - resp = session.ghrequest( "GET", next_link) + resp = session.ghrequest("GET", next_link) label_payloads.append(resp) else: break except Exception: traceback.print_exc() - - know_labels = [] for p in label_payloads: know_labels.extend([label["name"] for label in p.json()]) - print('known labels', know_labels) + print("known labels", know_labels) not_known_tags = [t for t in tags if t not in know_labels] known_tags = [t for t in tags if t in know_labels] - print('known tags', known_tags) - print('known labels', not_known_tags) + print("known tags", known_tags) + print("known labels", not_known_tags) # try to look at casing nk = [] known_lower_normal = {l.lower(): l for l in know_labels} - print('known labels lower', known_lower_normal) + print("known labels lower", known_lower_normal) for t in not_known_tags: target = known_lower_normal.get(t.lower()) - print('mapping t', t, target) + print("mapping t", t, target) if target: known_tags.append(t) else: - print('will not apply', t) + print("will not apply", t) nk.append(t) to_apply = known_tags @@ -991,7 +1149,7 @@ def get_next_link(req): comment_url, f"Aww {user}, I was not able to apply the following label(s): `{lf}`. Either " "because they are not existing labels on this repository or because you do not have the permission to apply these." - "I tried my best to guess by looking at the casing, but was unable to find matching labels.", + "I tried my best to guess by looking at the casing, but I was unable to find matching labels.", ) @@ -1003,9 +1161,7 @@ def untag(session, payload, arguments, local_config=None): num = payload.get("issue", payload.get("pull_request")).get("number") tags = [arg.strip() for arg in arguments.split(",")] name = "{name}" - url = "https://api.github.com/repos/{org}/{repo}/issues/{num}/labels/{name}".format( - **locals() - ) + url = f"https://api.github.com/repos/{org}/{repo}/issues/{num}/labels/{name}" no_untag = [] for tag in tags: try: @@ -1017,8 +1173,12 @@ def untag(session, payload, arguments, local_config=None): @write def migrate_issue_request( - *, session: Session, payload: dict, arguments: str, local_config=None -): + *, + session: Session, + payload: dict, + arguments: str, + local_config: Optional[dict] = None, +) -> Generator: """Todo: - Works through pagination of comments @@ -1034,9 +1194,7 @@ def migrate_issue_request( target_session = yield org_repo if not target_session: - session.post_comment( - payload["issue"]["comments_url"], "It appears that I can't do that" - ) + session.post_comment(payload["issue"]["comments_url"], "It appears that I can't do that") return issue_title = payload["issue"]["title"] @@ -1052,9 +1210,7 @@ def migrate_issue_request( if original_labels: available_labels = target_session.ghrequest( "GET", - "https://api.github.com/repos/{org}/{repo}/labels".format( - org=org, repo=repo - ), + f"https://api.github.com/repos/{org}/{repo}/labels", None, ).json() @@ -1081,9 +1237,7 @@ def migrate_issue_request( new_issue = new_response.json() new_comment_url = new_issue["comments_url"] - original_comments = session.ghrequest( - "GET", payload["issue"]["comments_url"], None - ).json() + original_comments = session.ghrequest("GET", payload["issue"]["comments_url"], None).json() for comment in original_comments: if comment["id"] == request_id: @@ -1097,9 +1251,7 @@ def migrate_issue_request( ) if not_set_labels: - body = "I was not able to apply the following label(s): %s " % ",".join( - not_set_labels - ) + body = "I was not able to apply the following label(s): %s " % ",".join(not_set_labels) target_session.post_comment(new_comment_url, body=body) session.post_comment( diff --git a/meeseeksdev/meeseeksbox/core.py b/meeseeksdev/meeseeksbox/core.py index edf5de4..5de402b 100644 --- a/meeseeksdev/meeseeksbox/core.py +++ b/meeseeksdev/meeseeksbox/core.py @@ -1,27 +1,21 @@ -import re +import base64 import hmac -import time import inspect - -import yaml -import base64 import json +import re +import time +from asyncio import Future +from concurrent.futures import ThreadPoolExecutor as Pool -import keen - -import tornado.web import tornado.httpserver import tornado.ioloop +import tornado.web +import yaml from tornado.ioloop import IOLoop +from yieldbreaker import YieldBreaker -from concurrent.futures import ThreadPoolExecutor as Pool - - -from .utils import Authenticator -from .utils import ACCEPT_HEADER_SYMMETRA from .scopes import Permission - -from yieldbreaker import YieldBreaker +from .utils import ACCEPT_HEADER_SYMMETRA, Authenticator, add_event, clear_caches green = "\033[0;32m" yellow = "\033[0;33m" @@ -30,19 +24,17 @@ pool = Pool(6) -import time - class Config: botname = None - integration_id = None + integration_id = -1 key = None botname = None at_botname = None integration_id = None webhook_secret = None - personnal_account_name = None - personnal_account_token = None + personal_account_name = None + personal_account_token = None def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -51,12 +43,10 @@ def validate(self): missing = [ attr for attr in dir(self) - if not attr.startswith("_") and getattr(self, attr) is None + if not attr.startswith("_") and getattr(self, attr) is None and attr != "key" ] if missing: - raise ValueError( - "The followingg configuration options are missing : {}".format(missing) - ) + raise ValueError(f"The following configuration options are missing : {missing}") return self @@ -73,21 +63,24 @@ def error(self, message): self.set_status(500) self.write({"status": "error", "message": message}) - def success(self, message="", payload={}): - self.write({"status": "success", "message": message, "data": payload}) + def success(self, message="", payload=None): + self.write({"status": "success", "message": message, "data": payload or {}}) class MainHandler(BaseHandler): def get(self): self.finish("No") -def _strip_please(c): - if c.startswith('please '): - return c[6:].lstrip() - else: - return c -def process_mentionning_comment(body, bot_re): +def _strip_extras(c): + if c.startswith("please "): + c = c[6:].lstrip() + if c.startswith("run "): + c = c[4:].lstrip() + return c + + +def process_mentioning_comment(body: str, bot_re: re.Pattern) -> list: """ Given a comment body and a bot name parse this into a tuple of (command, arguments) """ @@ -108,7 +101,7 @@ def process_mentionning_comment(body, bot_re): else: nl.append(bot_re.split(l)[-1].strip()) - command_args = [_strip_please(l).split(" ", 1) for l in nl] + command_args = [_strip_extras(l).split(" ", 1) for l in nl] command_args = [c if len(c) > 1 else [c[0], None] for c in command_args] return command_args @@ -121,7 +114,7 @@ def initialize(self, actions, config, auth, *args, **kwargs): super().initialize(*args, **kwargs) def get(self): - self.getfinish("Webhook alive and listening") + self.finish("Webhook alive and listening") def post(self): if self.config.forward_staging_url: @@ -141,26 +134,24 @@ def fn(req, url): "X-Hub-Signature", ) } - req = requests.Request( - "POST", url, headers=headers, data=req.body - ) + req = requests.Request("POST", url, headers=headers, data=req.body) prepared = req.prepare() with requests.Session() as s: - res = s.send(prepared) + res = s.send(prepared) # type:ignore[attr-defined] return res - except: + except Exception: import traceback traceback.print_exc() pool.submit(fn, self.request, self.config.forward_staging_url) - except: + except Exception: print(red + "failure to forward") import traceback traceback.print_exc() if "X-Hub-Signature" not in self.request.headers: - keen.add_event("attack", {"type": "no X-Hub-Signature"}) + add_event("attack", {"type": "no X-Hub-Signature"}) return self.error("WebHook not configured with secret") if not verify_signature( @@ -168,20 +159,13 @@ def fn(req, url): self.request.headers["X-Hub-Signature"], self.config.webhook_secret, ): - keen.add_event("attack", {"type": "wrong signature"}) - return self.error( - "Cannot validate GitHub payload with provided WebHook secret" - ) + add_event("attack", {"type": "wrong signature"}) + return self.error("Cannot validate GitHub payload with provided WebHook secret") payload = tornado.escape.json_decode(self.request.body) org = payload.get("repository", {}).get("owner", {}).get("login") if not org: - org = ( - payload.get("issue", {}) - .get("repository", {}) - .get("owner", {}) - .get("login") - ) + org = payload.get("issue", {}).get("repository", {}).get("owner", {}).get("login") print("org in issue", org) if payload.get("action", None) in [ @@ -192,23 +176,19 @@ def fn(req, url): "created", "submitted", ]: - keen.add_event("ignore_org_missing", {"edited": "reason"}) + add_event("ignore_org_missing", {"edited": "reason"}) else: - if hasattr(self.config, "org_whitelist") and ( - org not in self.config.org_whitelist - ): - keen.add_event("post", {"reject_organisation": org}) + if hasattr(self.config, "org_allowlist") and (org not in self.config.org_allowlist): + add_event("post", {"reject_organisation": org}) sender = payload.get("sender", {}).get("login", {}) - if hasattr(self.config, "user_blacklist") and ( - sender in self.config.user_blacklist - ): - keen.add_event("post", {"blocked_user": sender}) + if hasattr(self.config, "user_denylist") and (sender in self.config.user_denylist): + add_event("post", {"blocked_user": sender}) self.finish("Blocked user.") return action = payload.get("action", None) - keen.add_event("post", {"accepted_action": action}) + add_event("post", {"accepted_action": action}) unknown_repo = red + "" + normal repo = payload.get("repository", {}).get("full_name", unknown_repo) if repo == unknown_repo: @@ -244,9 +224,7 @@ def fn(req, url): "push", "create", }: - print( - f"(https://github.com/{repo}) Not handling event type `{event_type}` yet." - ) + print(f"(https://github.com/{repo}) Not handling event type `{event_type}` yet.") return self.finish() print(f"({repo}) No action available for the webhook :", event_type) @@ -254,13 +232,11 @@ def fn(req, url): @property def mention_bot_re(self): botname = self.config.botname - return re.compile("@?" + re.escape(botname) + "(?:\[bot\])?", re.IGNORECASE) + return re.compile("@?" + re.escape(botname) + r"(?:\[bot\])?", re.IGNORECASE) - def dispatch_action(self, type_, payload): + def dispatch_action(self, type_: str, payload: dict) -> Future: botname = self.config.botname - repo = payload.get("repository", {}).get( - "full_name", red + "" + normal - ) + repo = payload.get("repository", {}).get("full_name", red + "" + normal) # new issue/PR opened if type_ == "opened": issue = payload.get("issue", None) @@ -352,7 +328,7 @@ def dispatch_action(self, type_, payload): + f"(https://github.com/{repo}/pull/{num}) merged (action: {action}, merged:{merged}) by {login}" ) if merged_by: - description = "" + description = [] try: raw_labels = is_pr.get("labels", []) if raw_labels: @@ -364,27 +340,23 @@ def dispatch_action(self, type_, payload): raw_label.get("url", ""), override_accept_header=ACCEPT_HEADER_SYMMETRA, ).json() - label_desc = label.get("description", "") # apparently can still be none-like ? - if not label_desc: - label_desc = "" - description += "\n" + label_desc.replace("&", "\n") - except: + label_desc = label.get("description", "") or "" + description.append(label_desc.replace("&", "\n")) + except Exception: import traceback traceback.print_exc() milestone = is_pr.get("milestone", {}) if milestone: - e = milestone.get("description", "") - if not e: - e = "" - description += e - if ( - "on-merge:" in description - and is_pr["base"]["ref"] == "master" + description.append(milestone.get("description", "") or "") + description_str = "\n".join(description) + if "on-merge:" in description_str and is_pr["base"]["ref"] in ( + "master", + "main", ): did_backport = False - for description_line in description.splitlines(): + for description_line in description_str.splitlines(): line = description_line.strip() if line.startswith("on-merge:"): todo = line[len("on-merge:") :].strip() @@ -399,6 +371,13 @@ def dispatch_action(self, type_, payload): '"on-merge:" found in milestone description, but unable to parse command.', 'Is "on-merge:" on a separate line?', ) + print(description_str) + else: + print( + f'PR is not targeting main/master branch ({is_pr["base"]["ref"]}),' + 'or "on-merge:" not in milestone (or label) description:' + ) + print(description_str) else: print(f"({repo}) Hum, closed, PR but not merged") else: @@ -409,6 +388,7 @@ def dispatch_action(self, type_, payload): else: pass # print(f"({repo}) can't deal with `{type_}` yet") + return self.finish() # def _action_allowed(args): # """ @@ -421,7 +401,7 @@ def dispatch_action(self, type_, payload): # - If pull-request, the requester is the author. # """ - def dispatch_on_mention(self, body, payload, user): + def dispatch_on_mention(self, body: str, payload: dict, user: str) -> None: """ Core of the logic that let people require actions from the bot. @@ -473,16 +453,16 @@ def dispatch_on_mention(self, body, payload, user): print(user, "is legitimate author of this PR, letting commands go through") permission_level = session._get_permission(org, repo, user) - command_args = process_mentionning_comment(body, self.mention_bot_re) + command_args = process_mentioning_comment(body, self.mention_bot_re) for (command, arguments) in command_args: print(" :: treating", command, arguments) - keen.add_event( + add_event( "dispatch", { "mention": { "user": user, "organisation": org, - "repository": "{}/{}".format(org, repo), + "repository": f"{org}/{repo}", "command": command, } }, @@ -502,11 +482,7 @@ def user_can(user, command, repo, org, session): raise_for_status=False, ) except Exception: - print( - red - + "An error occurred getting repository config file." - + normal - ) + print(red + "An error occurred getting repository config file." + normal) import traceback traceback.print_exc() @@ -518,19 +494,13 @@ def user_can(user, command, repo, org, session): print(red + f"unknown status code {resp.status_code}" + normal) resp.raise_for_status() else: - conf = yaml.safe_load( - base64.decodebytes(resp.json()["content"].encode()) - ) - print( - green - + f"should test if {user} can {command} on {repo}/{org}" - + normal - ) + conf = yaml.safe_load(base64.decodebytes(resp.json()["content"].encode())) + print(green + f"should test if {user} can {command} on {repo}/{org}" + normal) # print(green + json.dumps(conf, indent=2) + normal) - if user in conf.get("blacklisted_users", []): + if user in conf.get("usr_denylist", []): return False, {} - + user_section = conf.get("users", {}).get(user, {}) custom_allowed_commands = user_section.get("can", []) @@ -584,9 +554,9 @@ def user_can(user, command, repo, org, session): traceback.print_exc() - has_scope = (permission_level.value >= handler.scope.value) + has_scope = permission_level.value >= handler.scope.value if has_scope: - local_config={} + local_config = {} if (has_scope) or ( is_legitimate_author @@ -611,9 +581,9 @@ def user_can(user, command, repo, org, session): gen = YieldBreaker(maybe_gen) for org_repo in gen: torg, trepo = org_repo.split("/") - session_id = self.auth.idmap.get(org_repo) - if session_id: - target_session = self.auth.session(session_id) + target_session = self.auth.get_session(org_repo) + + if target_session: # TODO, if PR, admin and request is on source repo, allows anyway. # we may need to also check allow edit from maintainer and provide # another decorator for safety. @@ -622,8 +592,7 @@ def user_can(user, command, repo, org, session): if target_session.has_permission( torg, trepo, user, Permission.write ) or ( - pr_origin_org_repo == org_repo - and allow_edit_from_maintainer + pr_origin_org_repo == org_repo and allow_edit_from_maintainer ): gen.send(target_session) else: @@ -657,8 +626,7 @@ def user_can(user, command, repo, org, session): class MeeseeksBox: def __init__(self, commands, config): - - keen.add_event("status", {"state": "starting"}) + add_event("status", {"state": "starting"}) self.commands = commands self.application = None self.config = config @@ -666,23 +634,21 @@ def __init__(self, commands, config): self.auth = Authenticator( self.config.integration_id, self.config.key, - self.config.personnal_account_token, - self.config.personnal_account_name, + self.config.personal_account_token, + self.config.personal_account_name, ) - self.auth._build_auth_id_mapping() def sig_handler(self, sig, frame): print(yellow, "Caught signal: %s, Shutting down..." % sig, normal) - keen.add_event("status", {"state": "stopping"}) - IOLoop.instance().add_callback(self.shutdown) + add_event("status", {"state": "stopping"}) + IOLoop.instance().add_callback_from_signal(self.shutdown) def shutdown(self): + print("in shutdown") self.server.stop() io_loop = IOLoop.instance() - deadline = time.time() + 10 - def stop_loop(): print(red, "stopping now...", normal) io_loop.stop() @@ -707,6 +673,13 @@ def start(self): ) self.server = tornado.httpserver.HTTPServer(self.application) - self.server.listen(self.port) - tornado.ioloop.IOLoop.instance().start() + + # Clear caches once per day. + callback_time_ms = 1000 * 60 * 60 * 24 + clear_cache_callback = tornado.ioloop.PeriodicCallback(clear_caches, callback_time_ms) + clear_cache_callback.start() + + loop = IOLoop.instance() + loop.add_callback(self.auth._build_auth_id_mapping) + loop.start() diff --git a/meeseeksdev/meeseeksbox/utils.py b/meeseeksdev/meeseeksbox/utils.py index 12c30bf..4f96d6f 100644 --- a/meeseeksdev/meeseeksbox/utils.py +++ b/meeseeksdev/meeseeksbox/utils.py @@ -1,22 +1,29 @@ """ Utility functions to work with github. """ -import jwt import datetime import json -import requests +import pipes import re +import shlex +import subprocess +from typing import Any, Dict, Optional, cast + +import jwt +import requests + +from .scopes import Permission green = "\033[0;32m" yellow = "\033[0;33m" red = "\033[0;31m" normal = "\033[0m" -from .scopes import Permission API_COLLABORATORS_TEMPLATE = ( "https://api.github.com/repos/{org}/{repo}/collaborators/{username}/permission" ) +ACCEPT_HEADER_V3 = "application/vnd.github.v3+json" ACCEPT_HEADER = "application/vnd.github.machine-man-preview" ACCEPT_HEADER_KORA = "json,application/vnd.github.korra-preview" ACCEPT_HEADER_SYMMETRA = "application/vnd.github.symmetra-preview+json" @@ -27,7 +34,26 @@ Pay attention to not relink things like foo#23 as they already point to a specific repository. """ -RELINK_RE = re.compile("(?:(?<=[:,\s])|(?<=^))(#\d+)\\b") +RELINK_RE = re.compile(r"(?:(?<=[:,\s])|(?<=^))(#\d+)\\b") + + +def add_event(*args): + """Attempt to add an event to keen, print the event otherwise""" + try: + import keen + + keen.add_event(*args) + except Exception: + print("Failed to log keen event:") + print(f" {args}") + + +def run(cmd, **kwargs): + """Print a command and then run it.""" + if isinstance(cmd, str): + cmd = shlex.split(cmd) + print(" ".join(map(pipes.quote, cmd))) + return subprocess.run(cmd, **kwargs) def fix_issue_body( @@ -40,38 +66,24 @@ def fix_issue_body( ): """ This, for now does only simple fixes, like link to the original issue. - This should be improved to quote mention of people """ - body = RELINK_RE.sub( - "{org}/{repo}\\1".format(org=original_org, repo=original_repo), body - ) + body = RELINK_RE.sub(f"{original_org}/{original_repo}\\1", body) - return ( - body - + """\n\n---- - \nOriginally opened as {org}/{repo}#{number} by @{reporter}, migration requested by @{requester} - """.format( - org=original_org, - repo=original_repo, - number=original_number, - reporter=original_poster, - requester=migration_requester, - ) - ) + return f"""{body}\n\n---- + \nOriginally opened as {original_org}/{original_repo}#{original_number} by @{original_poster}, migration requested by @{migration_requester} + """ def fix_comment_body(body, original_poster, original_url, original_org, original_repo): """ This, for now does only simple fixes, like link to the original comment. - + This should be improved to quote mention of people """ - body = RELINK_RE.sub( - "{org}/{repo}\\1".format(org=original_org, repo=original_repo), body - ) + body = RELINK_RE.sub(f"{original_org}/{original_repo}\\1", body) return """[`@{op}` commented]({original_url}): {body}""".format( op=original_poster, original_url=original_url, body=body @@ -80,21 +92,24 @@ def fix_comment_body(body, original_poster, original_url, original_org, original class Authenticator: def __init__( - self, integration_id, rsadata, personnal_account_token, personnal_account_name + self, + integration_id: int, + rsadata: Optional[str], + personal_account_token: Optional[str], + personal_account_name: Optional[str], ): self.since = int(datetime.datetime.now().timestamp()) self.duration = 60 * 10 self._token = None self.integration_id = integration_id self.rsadata = rsadata - self.personnal_account_token = personnal_account_token - self.personnal_account_name = personnal_account_name - # TODO: this mapping is built at startup, we should update it when we - # have new / deleted installations - self.idmap = {} + self.personal_account_token = personal_account_token + self.personal_account_name = personal_account_name + self.idmap: Dict[str, str] = {} + self._org_idmap: Dict[str, str] = {} self._session_class = Session - def session(self, installation_id): + def session(self, installation_id: str) -> "Session": """ Given and installation id, return a session with the right credentials """ @@ -104,46 +119,86 @@ def session(self, installation_id): self.integration_id, self.rsadata, installation_id, - self.personnal_account_token, - self.personnal_account_name, + self.personal_account_token, + self.personal_account_name, ) - def list_installations(self): - """ - Todo: Pagination - """ - # import json - # response = self._integration_authenticated_request( - # 'GET', "https://api.github.com/integration/installations") - # print(yellow+'list installation') - # print('HEADER', response.headers) - - response2 = self._integration_authenticated_request( - "GET", "https://api.github.com/app/installations" - ) - - # print(yellow+'list app installation') - # print('HEADER II', response2.headers) - # print('Content II', response2.json()) - return response2.json() + def get_session(self, org_repo): + """Given an org and repo, return a session with the right credentials.""" + # First try - see if we already have the auth. + if org_repo in self.idmap: + return self.session(self.idmap[org_repo]) + + # Next try - see if this is a newly authorized repo in an + # org that we've seen. + org, _ = org_repo.split("/") + if org in self._org_idmap: + self._update_installation(self._org_idmap[org]) + if org_repo in self.idmap: + return self.session(self.idmap[org_repo]) + + # TODO: if we decide to allow any org without an allowlist, + # we should make the org list dynamic. We would re-scan + # the list of installations here and update our mappings. + + def list_installations(self) -> Any: + """List the installations for the app.""" + installations = [] + + url = "https://api.github.com/app/installations" + while True: + response = self._integration_authenticated_request("GET", url) + installations.extend(response.json()) + if "next" in response.links: + url = response.links["next"]["url"] + continue + break + + return installations def _build_auth_id_mapping(self): """ Build an organisation/repo -> installation_id mappingg in order to be able to do cross repository operations. """ - - installations = self.list_installations() - for installation in installations: - iid = installation["id"] - session = self.session(iid) - repositories = session.ghrequest( - "GET", installation["repositories_url"], json=None - ).json() - for repo in repositories["repositories"]: - self.idmap[repo["full_name"]] = iid - - def _integration_authenticated_request(self, method, url): + if not self.rsadata: + print("Skipping auth_id_mapping build since there is no B64KEY set") + return + + self._installations = self.list_installations() + for installation in self._installations: + self._update_installation(installation) + + def _update_installation(self, installation): + print("Updating installations", installation) + iid = installation["id"] + print("... making a session", iid) + session = self.session(iid) + try: + # Make sure we get all pages. + url = installation["repositories_url"] + while True: + res = session.ghrequest("GET", url) + repositories = res.json() + for repo in repositories["repositories"]: + print( + "Mapping repo to installation:", + repo["full_name"], + repo["owner"]["login"], + iid, + ) + self.idmap[repo["full_name"]] = iid + self._org_idmap[repo["owner"]["login"]] = iid + if "next" in res.links: + url = res.links["next"]["url"] + continue + break + + except Forbidden: + print("Forbidden for", iid) + return + + def _integration_authenticated_request(self, method, url, json=None): self.since = int(datetime.datetime.now().timestamp()) payload = dict( { @@ -153,18 +208,23 @@ def _integration_authenticated_request(self, method, url): } ) + assert self.rsadata is not None tok = jwt.encode(payload, key=self.rsadata, algorithm="RS256") headers = { - "Authorization": "Bearer {}".format(tok.decode()), - "Accept": ACCEPT_HEADER, + "Authorization": f"Bearer {tok}", + "Accept": ACCEPT_HEADER_V3, "Host": "api.github.com", "User-Agent": "python/requests", } - req = requests.Request(method, url, headers=headers) + req = requests.Request(method, url, headers=headers, json=json) prepared = req.prepare() with requests.Session() as s: - return s.send(prepared) + return s.send(prepared) # type:ignore[attr-defined] + + +class Forbidden(Exception): + pass class Session(Authenticator): @@ -173,30 +233,38 @@ def __init__( integration_id, rsadata, installation_id, - personnal_account_token, - personnal_account_name, + personal_account_token, + personal_account_name, ): - super().__init__( - integration_id, rsadata, personnal_account_token, personnal_account_name - ) + super().__init__(integration_id, rsadata, personal_account_token, personal_account_name) self.installation_id = installation_id - def token(self): + def token(self) -> str: now = datetime.datetime.now().timestamp() if (now > self.since + self.duration - 60) or (self._token is None): self.regen_token() + assert self._token is not None return self._token - def regen_token(self): + def regen_token(self) -> None: method = "POST" url = f"https://api.github.com/app/installations/{self.installation_id}/access_tokens" resp = self._integration_authenticated_request(method, url) + if resp.status_code == 403: + raise Forbidden(self.installation_id) + try: self._token = json.loads(resp.content.decode())["token"] - except: + except Exception: raise ValueError(resp.content, url) - def personal_request(self, method, url, json=None, raise_for_status=True): + def personal_request( + self, + method: str, + url: str, + json: Optional[dict] = None, + raise_for_status: bool = True, + ) -> requests.Response: """ Does a request but using the personal account name and token """ @@ -205,7 +273,7 @@ def personal_request(self, method, url, json=None, raise_for_status=True): def prepare(): headers = { - "Authorization": "token {}".format(self.personnal_account_token), + "Authorization": f"token {self.personal_account_token}", "Host": "api.github.com", "User-Agent": "python/requests", } @@ -213,23 +281,23 @@ def prepare(): return req.prepare() with requests.Session() as s: - response = s.send(prepare()) + response = s.send(prepare()) # type:ignore[attr-defined] if response.status_code == 401: self.regen_token() - response = s.send(prepare()) + response = s.send(prepare()) # type:ignore[attr-defined] if raise_for_status: response.raise_for_status() - return response + return response # type:ignore[no-any-return] def ghrequest( self, - method, - url, - json=None, + method: str, + url: str, + json: Optional[dict] = None, *, - override_accept_header=None, - raise_for_status=True, - ): + override_accept_header: Optional[str] = None, + raise_for_status: Optional[bool] = True, + ) -> requests.Response: accept = ACCEPT_HEADER if override_accept_header: accept = override_accept_header @@ -237,27 +305,27 @@ def ghrequest( def prepare(): atk = self.token() headers = { - "Authorization": "Bearer {}".format(atk), + "Authorization": f"Bearer {atk}", "Accept": accept, "Host": "api.github.com", "User-Agent": "python/requests", } + print(f"Making a {method} call to {url}") req = requests.Request(method, url, headers=headers, json=json) return req.prepare() with requests.Session() as s: - response = s.send(prepare()) + response = s.send(prepare()) # type:ignore[attr-defined] if response.status_code == 401: + print("Unauthorized, regen token") self.regen_token() - response = s.send(prepare()) + response = s.send(prepare()) # type:ignore[attr-defined] if raise_for_status: response.raise_for_status() rate_limit = response.headers.get("X-RateLimit-Limit", -1) rate_remaining = response.headers.get("X-RateLimit-Limit", -1) if rate_limit: - repo_name_list = [ - k for k, v in self.idmap.items() if v == self.installation_id - ] + repo_name_list = [k for k, v in self.idmap.items() if v == self.installation_id] repo_name = "no-repo" if len(repo_name_list) == 1: repo_name = repo_name_list[0] @@ -266,9 +334,7 @@ def prepare(): else: repo_name = "multiple-matches" - import keen - - keen.add_event( + add_event( "gh-rate", { "limit": int(rate_limit), @@ -276,12 +342,13 @@ def prepare(): "installation": repo_name, }, ) - return response + return response # type:ignore[no-any-return] - def _get_permission(self, org, repo, username): + def _get_permission(self, org: str, repo: str, username: str) -> Permission: get_collaborators_query = API_COLLABORATORS_TEMPLATE.format( org=org, repo=repo, username=username ) + print("_get_permission") resp = self.ghrequest( "GET", get_collaborators_query, @@ -291,20 +358,21 @@ def _get_permission(self, org, repo, username): resp.raise_for_status() permission = resp.json()["permission"] print("found permission", permission, "for user ", username, "on ", org, repo) - return getattr(Permission, permission) + return cast(Permission, getattr(Permission, permission)) - def has_permission(self, org, repo, username, level=None): - """ - """ + def has_permission( + self, org: str, repo: str, username: str, level: Optional[Permission] = None + ) -> bool: + """ """ if not level: level = Permission.none return self._get_permission(org, repo, username).value >= level.value - def post_comment(self, comment_url, body): + def post_comment(self, comment_url: str, body: str) -> None: self.ghrequest("POST", comment_url, json={"body": body}) - def get_collaborator_list(self, org, repo): + def get_collaborator_list(self, org: str, repo: str) -> Optional[Any]: get_collaborators_query = "https://api.github.com/repos/{org}/{repo}/collaborators".format( org=org, repo=repo ) @@ -313,11 +381,19 @@ def get_collaborator_list(self, org, repo): return resp.json() else: resp.raise_for_status() + return None def create_issue( - self, org: str, repo: str, title: str, body: str, *, labels=None, assignees=None - ): - arguments = {"title": title, "body": body} + self, + org: str, + repo: str, + title: str, + body: str, + *, + labels: Optional[list] = None, + assignees: Optional[list] = None, + ) -> requests.Response: + arguments: dict = {"title": title, "body": body} if labels: if type(labels) in (list, tuple): @@ -333,6 +409,14 @@ def create_issue( return self.ghrequest( "POST", - "https://api.github.com/repos/{}/{}/issues".format(org, repo), + f"https://api.github.com/repos/{org}/{repo}/issues", json=arguments, ) + + +def clear_caches(): + """Clear local caches""" + print("\n\n====Clearing all caches===") + run("pip cache purge") + run("pre-commit clean") + print("====Finished clearing caches===\n\n") diff --git a/meeseeksdev/tests/test_misc.py b/meeseeksdev/tests/test_misc.py index 66b9f18..b474cf3 100644 --- a/meeseeksdev/tests/test_misc.py +++ b/meeseeksdev/tests/test_misc.py @@ -1,23 +1,33 @@ -from ..meeseeksbox.core import process_mentionning_comment -import textwrap import re +import textwrap +from ..meeseeksbox.core import process_mentioning_comment def test1(): - botname = 'meeseeksdev' - reg = re.compile("@?" + re.escape(botname) + "(?:\[bot\])?", re.IGNORECASE) + botname = "meeseeksdev" + reg = re.compile("@?" + re.escape(botname) + r"(?:\[bot\])?", re.IGNORECASE) - assert process_mentionning_comment(textwrap.dedent(''' + assert ( + process_mentioning_comment( + textwrap.dedent( + """ @meeseeksdev nothing @meeseeksdev[bot] do nothing meeseeksdev[bot] do something - - - '''), reg) == [['nothing', None], - ['do', 'nothing'], - ['do', 'something']] - - - - + @meeseeksdev please nothing + @meeseeksdev run something + + + """ + ), + reg, + ) + == [ + ["nothing", None], + ["do", "nothing"], + ["do", "something"], + ["nothing", None], + ["something", None], + ] + ) diff --git a/meeseeksdev/tests/test_webhook.py b/meeseeksdev/tests/test_webhook.py new file mode 100644 index 0000000..de886c3 --- /dev/null +++ b/meeseeksdev/tests/test_webhook.py @@ -0,0 +1,58 @@ +import hmac + +import pytest +import tornado.web + +from ..meeseeksbox.core import Authenticator, Config, WebHookHandler + +commands: dict = {} + +config = Config( + integration_id=100, + key=None, + personal_account_token="foo", + personal_account_name="bar", + forward_staging_url="", + webhook_secret="foo", +) + +auth = Authenticator( + config.integration_id, + config.key, + config.personal_account_token, + config.personal_account_name, +) + +application = tornado.web.Application( + [ + ( + r"/", + WebHookHandler, + { + "actions": commands, + "config": config, + "auth": auth, + }, + ), + ] +) + + +@pytest.fixture +def app(): + return application + + +async def test_get(http_server_client): + response = await http_server_client.fetch("/") + assert response.code == 200 + + +async def test_post(http_server_client): + body = "{}" + secret = config.webhook_secret + assert secret is not None + sig = "sha1=" + hmac.new(secret.encode("utf8"), body.encode("utf8"), "sha1").hexdigest() + headers = {"X-Hub-Signature": sig} + response = await http_server_client.fetch("/", method="POST", body=body, headers=headers) + assert response.code == 200 diff --git a/package.json b/package.json new file mode 100644 index 0000000..40f0e07 --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "name": "meeseeksdev", + "version": "1.0.0", + "description": "stub file for node.js buildpack", + "private": true +} diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..f410c11 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +-r ./requirements.txt +pytest>=6.0 +pytest-tornasync diff --git a/requirements.txt b/requirements.txt index 34310ed..2eccb0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ -tornado==6.0.3 -requests==2.23.0 -pyjwt==1.7.1 -gitpython==2.1.12 +black==24.3.0 +cryptography==46.0.5 +gitpython==3.1.41 +keen==0.7 +mock==4.0 +pip==26.0 +pre-commit==2.17 +pyjwt==2.4.0 +pyyaml==6.0 +requests==2.32.4 there==0.0.9 -mock==3.0.5 -cryptography==2.8 +tornado==6.5.1 yieldbreaker==0.0.1 -black -pyyaml==5.3 -keen diff --git a/runtime.txt b/runtime.txt index 6f651a3..74d315a 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.7.3 +python-3.12.4