diff options
-rwxr-xr-x | scripts/cgit-last-modified.py | 30 | ||||
-rwxr-xr-x | scripts/ia-upload-warc | 8 |
2 files changed, 0 insertions, 38 deletions
diff --git a/scripts/cgit-last-modified.py b/scripts/cgit-last-modified.py deleted file mode 100755 index 46d7720..0000000 --- a/scripts/cgit-last-modified.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 - -import git -import os -from datetime import datetime, timezone, timedelta - -REPOS = "/var/www/git.jordan.im/repositories" - -for x in os.listdir(REPOS): - if not os.path.isdir(os.path.join(REPOS, x)): - continue - - print(f"processing {x}...") - - try: - repo = git.Repo(os.path.join(REPOS, x)) - for remote in repo.remotes: - remote.fetch() - md = repo.head.commit.committed_datetime - md = md.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") - except Exception as e: - sys.stderr.write(e) - continue - - d = os.path.join(REPOS, x, "info/web/") - os.makedirs(d, exist_ok=True) - - f = open(os.path.join(d, "last-modified"), "w") - f.write(md) - f.close() diff --git a/scripts/ia-upload-warc b/scripts/ia-upload-warc deleted file mode 100755 index 789d1a8..0000000 --- a/scripts/ia-upload-warc +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -title="$(echo $1 | sed 's/\(-[0-9][0-9][0-9][0-9]-[0-9]\+-[0-9]\+.*\)//g')" -date="$(echo $1 | sed -n 's/.*\([0-9][0-9][0-9][0-9]-[0-9]\+-[0-9]\+\).*/\1/p')" -id=$title-$date -start_url="$(cat $1/start_url)" - -ia upload $id $1/*.gz $1/*.cdx --metadata="mediatype:web" --metadata="date:$date" --metadata="title:$title" --metadata="description:recursive crawl of $start_url using grab-site/wpull, taken on $date" --metadata="source:$title" |