From ae0588e5693f5e723db656cee51fa838cc08d712 Mon Sep 17 00:00:00 2001 From: Enrico Ludwig Date: Sat, 13 Jul 2024 16:16:38 +0200 Subject: [PATCH] Moved scripts to separate folders, started working on GitLab to Gitea migration script --- .gitignore | 1 + LICENSE | 2 +- README.md | 10 +- .../borgmatic_setup.sh | 0 gitlab2gitea/gitlab2gitea.py | 356 ++++++++++++++++++ ufw_beautifier.py => ufw/ufw_beautifier.py | 0 6 files changed, 363 insertions(+), 6 deletions(-) create mode 100644 .gitignore rename borgmatic_setup.sh => borgmatic/borgmatic_setup.sh (100%) create mode 100644 gitlab2gitea/gitlab2gitea.py rename ufw_beautifier.py => ufw/ufw_beautifier.py (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/LICENSE b/LICENSE index 02c3bec..2713bb2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 ZionNetworks +Copyright (c) 2024 Zion Networks UG Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/README.md b/README.md index af88a9f..0b2675c 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ We're planning to release a growing amount of open source software, that is free ### What scripts can be found here? -| Name | Description | License | Current Version | Written in | Supported Distros | File | -|------|-------------|---------|-----------------|------------|-------------------|------| -| [Borgmatic Setup Tool](#borgmatic-setup-tool) | If you plan to use borg as backup solution, you should also take a look at [borgmatic](https://torsion.org/borgmatic/). It's a Python wrapper for the award winning backup tool [borgbackup](https://borgbackup.readthedocs.io/en/stable/index.html) that simplifies creating secure and reliable backups even more. You can even store your configurations in files. This script will do the setup for you to. | [MIT]([LICENSE](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/LICENSE)) | v1.2.1 | Bash | Debian and derivates | [bormatic_setup.sh](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh) | +| Name | Description | License | Current Version | Written in | Supported Distros | File | +| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | --------------- | ---------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| [Borgmatic Setup Tool](#borgmatic-setup-tool) | If you plan to use borg as backup solution, you should also take a look at [borgmatic](https://torsion.org/borgmatic/). It's a Python wrapper for the award winning backup tool [borgbackup](https://borgbackup.readthedocs.io/en/stable/index.html) that simplifies creating secure and reliable backups even more. You can even store your configurations in files. This script will do the setup for you to. | [MIT]([LICENSE](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/LICENSE)) | v1.2.1 | Bash | Debian and derivates | [bormatic_setup.sh](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh) | #### Borgmatic Setup Tool @@ -68,8 +68,8 @@ If you want the script to also upgrade the packages on your system, set the `--u ###### Example: Backup local configurations and home directories 1. Get the script via `wget` or `curl`: - 1. `wget https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh` - 2. `curl -O https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh` + 1. `wget https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh` + 2. `curl -O https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh` 2. Set the execution flag using `chmod +x borgmatic_setup.sh` 3. Run the script: diff --git a/borgmatic_setup.sh b/borgmatic/borgmatic_setup.sh similarity index 100% rename from borgmatic_setup.sh rename to borgmatic/borgmatic_setup.sh diff --git a/gitlab2gitea/gitlab2gitea.py b/gitlab2gitea/gitlab2gitea.py new file mode 100644 index 0000000..d5ccd23 --- /dev/null +++ b/gitlab2gitea/gitlab2gitea.py @@ -0,0 +1,356 @@ +# Description: Script to migrate repositories from GitLab to Gitea +# Author: Enrico Ludwig +# Version: 1.0 +# Date: 2021-01-06 +# +# Precedence of settings: arguments > .env file > environment variables > default values +# +# Usage: +# python3 gitlab2gitea.py [options] +# +# --gitlab-token GitLab access token +# --gitlab-url GitLab URL +# --gitlab-api-version GitLab API version (default: v4) +# --gitea-token Gitea access token +# --gitea-url Gitea URL +# --gitea-api-version Gitea API version (default: v1) +# +# --quiet Enable quiet mode (default: False) +# --debug Enable debug mode (default: False) +# --dry-run Enable dry-run mode (default: False) +# --log-file Log file (default: gitlab2gitea.log) +# If not set, logs will be only printed to stdout +# --append-log Append log file (default: False) +# +# Example: +# python3 gitlab2gitea.py \ +# --gitlab-url \ +# --gitlab-token \ +# --gitea-url \ +# --gitea-token + +# Settings - GitLab + +GITLAB_URL = '' +GITLAB_TOKEN = '' +GITLAB_API_VERSION = 'v4' + +# Settings - Gitea + +GITEA_URL = '' +GITEA_TOKEN = '' +GITEA_API_VERSION = 'v1' + +# Settings - General + +DEBUG = False +DRY_RUN = False +LOG_FILE = 'gitlab2gitea.log' +APPEND_LOG = False +QUIET = False + +# Imports + +import os +import argparse +import requests + +# Set cwd to script directory + +os.chdir(os.path.dirname(os.path.realpath(__file__))) + +# Read environment variables +if 'GITLAB_URL' in os.environ: + GITLAB_URL = os.environ['GITLAB_URL'] + +if 'GITLAB_TOKEN' in os.environ: + GITLAB_TOKEN = os.environ['GITLAB_TOKEN'] + +if 'GITLAB_API_VERSION' in os.environ: + GITLAB_API_VERSION = os.environ['GITLAB_API_VERSION'] + +if 'GITEA_URL' in os.environ: + GITEA_URL = os.environ['GITEA_URL'] + +if 'GITEA_TOKEN' in os.environ: + GITEA_TOKEN = os.environ['GITEA_TOKEN'] + +if 'GITEA_API_VERSION' in os.environ: + GITEA_API_VERSION = os.environ['GITEA_API_VERSION'] + +if 'QUIET' in os.environ: + QUIET = bool(os.environ['QUIET']) + +if 'DEBUG' in os.environ: + DEBUG = bool(os.environ['DEBUG']) + +if 'DRY_RUN' in os.environ: + DRY_RUN = bool(os.environ['DRY_RUN']) + +if 'LOG_FILE' in os.environ: + LOG_FILE = os.environ['LOG_FILE'] + +if 'APPEND_LOG' in os.environ: + APPEND_LOG = bool(os.environ['APPEND_LOG']) + +# Read .env file if exists and override environment variables + +if os.path.exists('.env'): + with open('.env', 'r') as env_file: + for line in env_file: + # skip comments, empty lines and lines without '=' + if line.startswith('#') or not '=' in line: + continue + + key, value = line.strip().split('=') + + if key == 'GITLAB_API_VERSION': + GITLAB_API_VERSION = value + + if key == 'GITLAB_URL': + GITLAB_URL = value + + if key == 'GITLAB_TOKEN': + GITLAB_TOKEN = value + + if key == 'GITEA_URL': + GITEA_URL = value + + if key == 'GITEA_TOKEN': + GITEA_TOKEN = value + + if key == 'GITEA_API_VERSION': + GITEA_API_VERSION = value + + if key == 'QUIET': + QUIET = bool(value) + + if key == 'DEBUG': + DEBUG = bool(value) + + if key == 'DRY_RUN': + DRY_RUN = bool(value) + + if key == 'LOG_FILE': + LOG_FILE = value + + if key == 'APPEND_LOG': + APPEND_LOG = bool(value) + +# Read arguments and override environment variables + +parser = argparse.ArgumentParser(description='Script to migrate repositories from GitLab to Gitea') + +parser.add_argument('--gitlab-api-version', help='GitLab API version', default='v4') +parser.add_argument('--gitlab-token', help='GitLab access token') +parser.add_argument('--gitea-token', help='Gitea access token') +parser.add_argument('--gitlab-url', help='GitLab URL') +parser.add_argument('--gitea-url', help='Gitea URL') +parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1') +parser.add_argument('--quiet', help='Enable quiet mode', action='store_true') +parser.add_argument('--debug', help='Enable debug mode', action='store_true') +parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true') +parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log') +parser.add_argument('--append-log', help='Append log file', action='store_true') + +args = parser.parse_args() + +if args.gitlab_api_version: + GITLAB_API_VERSION = args.gitlab_api_version + +if args.gitlab_token: + GITLAB_TOKEN = args.gitlab_token + +if args.gitlab_url: + GITLAB_URL = args.gitlab_url + +if args.gitea_token: + GITEA_TOKEN = args.gitea_token + +if args.gitea_url: + GITEA_URL = args.gitea_url + +if args.gitea_api_version: + GITEA_API_VERSION = args.gitea_api_version + +if args.quiet: + QUIET = True + +if args.debug: + DEBUG = True + +if args.dry_run: + DRY_RUN = True + +if args.log_file: + LOG_FILE = args.log_file + +if args.append_log: + APPEND_LOG = True + +# Remove trailing slashes from URLs + +GITLAB_URL = GITLAB_URL.rstrip('/') +GITEA_URL = GITEA_URL.rstrip('/') + +# Internal functions + +def _debug(message): + if DEBUG: + print('\033[1m\033[34m[DBG]\033[0m', message) + + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[DBG] {message}\n') + +def _info(message): + print('\033[1m\033[32m[INF]\033[0m', message) + + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[INF] {message}\n') + +def _warn(message): + print('\033[1m\033[33m[WRN]\033[0m', message) + + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[WRN] {message}\n') + +def _error(message): + print('\033[1m\033[31m[ERR]\033[0m', message) + + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[ERR] {message}\n') + +# PROGRAM + +# Endpoint: GET /api/{GITLAB_API_VERSION}/version +def check_gitlab(): + _debug(f'REQUEST: {GITLAB_URL}/api/{GITLAB_API_VERSION}/version') + + response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/version', headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'Bearer {GITLAB_TOKEN}' + }) + + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'GitLab endpoint test failed: {response_message}') + else: + _info(f'GitLab endpoint version: {response.json()["version"]}') + +# Endpoint: GET /api/{GITEA_API_VERSION}/version +def check_gitea(): + _debug(f'REQUEST: {GITEA_URL}/api/{GITEA_API_VERSION}/version') + + response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/version', headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'token {GITEA_TOKEN}' + }) + + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'Gitea endpoint test failed: {response_message}') + else: + _info(f'Gitea endpoint version: {response.json()["version"]}') + +# Endpoint: GET /api/{GITLAB_API_VERSION}/groups +def get_gitlab_groups() -> list: + pass + +# Endpoint: GET /api/{GITLAB_API_VERSION}/users +def get_gitlab_users() -> list: + pass + +# Endpoint: GET /api/{GITLAB_API_VERSION}/projects +def get_gitlab_projects() -> list: + pass + +# Endpoint: GET /api/{GITEA_API_VERSION}/orgs +def get_gitea_groups() -> list: + pass + +# Endpoint: GET /api/{GITEA_API_VERSION}/users +def get_gitea_users() -> list: + pass + +# Endpoint: GET /api/{GITEA_API_VERSION}/repos +def get_gitea_projects() -> list: + pass + +def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict: + pass + +def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict: + pass + +def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict: + pass + +def run_migration(): + + _info('Migrating GitLab groups...') + + gitlab_groups = get_gitlab_groups() + gitea_groups = get_gitea_groups() + + cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) + + _info('Migrating GitLab users...') + + gitlab_users = get_gitlab_users() + gitea_users = get_gitea_users() + + cmp_gitlab_gitea_users(gitlab_users, gitea_users) + + _info('Migrating GitLab projects...') + + gitlab_projects = get_gitlab_projects() + gitea_projects = get_gitea_projects() + + cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects) + + + +def main(): + if DRY_RUN: + _warn('Running in dry-run mode!') + + if not GITLAB_TOKEN: + _error('GitLab access token is missing!') + return + + if not GITEA_TOKEN: + _error('Gitea access token is missing!') + return + + if LOG_FILE and not APPEND_LOG: + with open(LOG_FILE, 'w') as log_file: + log_file.write('') + + _debug(f'GitLab URL: {GITLAB_URL}') + _debug(f'Gitea URL: {GITEA_URL}') + _debug(f'Logging to: {LOG_FILE}') + + _info('Testing endpoints...') + + try: + check_gitlab() + check_gitea() + except Exception as e: + _error(f'An error occurred: {e}') + return + + _info('Starting migration...') + + try: + run_migration() + except Exception as e: + _error(f'An error occurred: {e}') + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/ufw_beautifier.py b/ufw/ufw_beautifier.py similarity index 100% rename from ufw_beautifier.py rename to ufw/ufw_beautifier.py