From 3e78acc804ac53f75dee87f4edd499c6fad3a277 Mon Sep 17 00:00:00 2001 From: Jordan Date: Tue, 13 Sep 2022 21:36:22 -0700 Subject: initial commit --- README | 12 ++++++ UNLICENSE | 24 ++++++++++++ koya | 133 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 14 +++++++ 4 files changed, 183 insertions(+) create mode 100644 README create mode 100644 UNLICENSE create mode 100755 koya create mode 100644 setup.py diff --git a/README b/README new file mode 100644 index 0000000..73eb14f --- /dev/null +++ b/README @@ -0,0 +1,12 @@ +koya: archive git repositories from sourcehut user accounts + +usage: koya.py [-h] [--origin ORIGIN] --token TOKEN [--delay DELAY] users [users ...] + +positional arguments: + users username(s) to archive + +options: + -h, --help show this help message and exit + --origin ORIGIN sourcehut origin URL (default: https://git.sr.ht/) + --token TOKEN account oauth access token (default: None) + --delay DELAY delay between origin requests (seconds) (default: 1) diff --git a/UNLICENSE b/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/koya b/koya new file mode 100755 index 0000000..826b344 --- /dev/null +++ b/koya @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""koya + +Usage: + koya --token user1 user2... + +Options: + -h, --help show this help message and exit + --origin ORIGIN sourcehut origin URL (default: https://git.sr.ht/) + --token TOKEN account oauth access token (default: None) + --delay DELAY delay between origin requests (seconds) (default: 1) +""" + +import argparse +import json +import logging +import os +import subprocess +import sys +import time +from pathlib import Path +from urllib.parse import urljoin, urlparse, quote_plus + +import requests + +logging.basicConfig( + stream=sys.stdout, format='%(asctime)s %(message)s', + datefmt='%m/%d/%Y %H:%M:%S') +log = logging.getLogger('koya') +log.setLevel(logging.DEBUG) + +class Koya: + def __init__(self, origin, token, delay): + self.origin = origin + self.token = token + self.delay = delay + + self.session = requests.Session() + self.session.headers.update({"User-Agent": "koya/1.0 git.jordan.im/koya"}) + + def list_repos(self, user): + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.token}"} + repos = [] + cursor_obj = "" + while True: + graphql = f""" + query {{ + user(username: "{user}") {{ + repositories({cursor_obj}) {{ + cursor + results {{ + name + }} + }} + }} + }}""" + try: + u = urljoin(self.origin, "/query") + r = self.session.post(u, headers=headers, json={"query": graphql}) + r.raise_for_status() + except Exception as err: + log.error(err) + return + + res = json.loads(r.text) + if not res["data"].get("user"): + return repos + + data = res["data"]["user"]["repositories"] + repos.extend(data["results"]) + + cursor = data.get("cursor") + if cursor: + cursor_obj = f'''cursor: "{cursor}"''' + else: + return repos + + log.info(f"Discovered {len(data['results'])} repositories...") + time.sleep(self.delay) + +def multi_urljoin(*parts): + ret = urljoin(parts[0], "/".join( + quote_plus(part.strip("/"), safe="/") for part in parts[1:])) + return ret + +if __name__ == "__main__": + DESC = "koya: archive git repositories from sourcehut user accounts" + parser = argparse.ArgumentParser( + description=DESC, + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + "users", nargs="+", default=os.getcwd(), help="username(s) to archive") + parser.add_argument( + "--origin", dest="origin", type=str, action="store", + default="https://git.sr.ht/", required=False, + help="sourcehut origin URL") + parser.add_argument( + "--token", dest="token", type=str, action="store", + help="account oauth access token", required=True) + parser.add_argument( + "--delay", dest="delay", type=int, action="store", default=1, + help="delay between origin requests (seconds)") + args=parser.parse_args() + + host = urlparse(args.origin).hostname + if not host: + log.error("Unable to parse hostname from origin, exiting...") + sys.exit() + + koya = Koya(args.origin, args.token, args.delay) + for user in args.users: + repos = koya.list_repos(user) + if not repos: + log.error(f"No repositories discovered, skipping {user}...") + continue + + for repo in repos: + repo_path = os.path.join(host, user, repo["name"]) + if os.path.exists(repo_path): + fetch = ["git", "fetch", "--all", "--force", "--tags", "--prune"] + subprocess.run(fetch, cwd=repo_path) + + time.sleep(args.delay) + else: + Path(repo_path).mkdir(parents=True, exist_ok=True) + + remote_url = multi_urljoin(args.origin, "~"+user, repo["name"]) + clone = ["git", "clone", remote_url, repo_path] + subprocess.run(clone) + + time.sleep(args.delay) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6b35b3f --- /dev/null +++ b/setup.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 + +from setuptools import setup + +setup( + name="koya", + author="Jordan", + author_email="me@jordan.im", + url="https://git.jordan.im/koya", + description="Archive git repositories from sourcehut user accounts", + license="Unlicense", + scripts=["koya"], + install_requires=["requests"] +) -- cgit v1.2.3-54-g00ecf