From 0c1e91d6562bedda6e4b3e88d0ff3dfe8f51f318 Mon Sep 17 00:00:00 2001 From: Enrico Ludwig Date: Sat, 13 Jul 2024 17:15:10 +0200 Subject: [PATCH] Further work on gitlab2gitea --- gitlab2gitea/gitlab2gitea.py | 253 +++++++++++++++++++++++++++++++++-- 1 file changed, 239 insertions(+), 14 deletions(-) diff --git a/gitlab2gitea/gitlab2gitea.py b/gitlab2gitea/gitlab2gitea.py index d5ccd23..67d89f1 100644 --- a/gitlab2gitea/gitlab2gitea.py +++ b/gitlab2gitea/gitlab2gitea.py @@ -15,8 +15,12 @@ # --gitea-url Gitea URL # --gitea-api-version Gitea API version (default: v1) # +# --create-missing-groups Create missing groups on Gitea (default: False) +# --create-missing-users Create missing users on Gitea (default: False) +# # --quiet Enable quiet mode (default: False) # --debug Enable debug mode (default: False) +# --trace Enable trace mode (default: False) # --dry-run Enable dry-run mode (default: False) # --log-file Log file (default: gitlab2gitea.log) # If not set, logs will be only printed to stdout @@ -41,9 +45,15 @@ GITEA_URL = '' GITEA_TOKEN = '' GITEA_API_VERSION = 'v1' +# Settings - General Repository + +CREATE_MISSING_GROUPS = False +CREATE_MISSING_USERS = False + # Settings - General DEBUG = False +TRACE = False DRY_RUN = False LOG_FILE = 'gitlab2gitea.log' APPEND_LOG = False @@ -52,6 +62,7 @@ QUIET = False # Imports import os +import sys import argparse import requests @@ -78,12 +89,21 @@ if 'GITEA_TOKEN' in os.environ: if 'GITEA_API_VERSION' in os.environ: GITEA_API_VERSION = os.environ['GITEA_API_VERSION'] +if 'CREATE_MISSING_GROUPS' in os.environ: + CREATE_MISSING_GROUPS = bool(os.environ['CREATE_MISSING_GROUPS']) + +if 'CREATE_MISSING_USERS' in os.environ: + CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS']) + if 'QUIET' in os.environ: QUIET = bool(os.environ['QUIET']) if 'DEBUG' in os.environ: DEBUG = bool(os.environ['DEBUG']) +if 'TRACE' in os.environ: + TRACE = bool(os.environ['TRACE']) + if 'DRY_RUN' in os.environ: DRY_RUN = bool(os.environ['DRY_RUN']) @@ -121,15 +141,29 @@ if os.path.exists('.env'): if key == 'GITEA_API_VERSION': GITEA_API_VERSION = value + + if key == 'CREATE_MISSING_GROUPS': + if value.lower() == 'true' or value == '1': + CREATE_MISSING_GROUPS = True + + if key == 'CREATE_MISSING_USERS': + if value.lower() == 'true' or value == '1': + CREATE_MISSING_USERS = True if key == 'QUIET': QUIET = bool(value) if key == 'DEBUG': - DEBUG = bool(value) + if value.lower() == 'true' or value == '1': + DEBUG = True + + if key == 'TRACE': + if value.lower() == 'true' or value == '1': + TRACE = True if key == 'DRY_RUN': - DRY_RUN = bool(value) + if value.lower() == 'true' or value == '1': + DRY_RUN = True if key == 'LOG_FILE': LOG_FILE = value @@ -147,8 +181,11 @@ parser.add_argument('--gitea-token', help='Gitea access token') parser.add_argument('--gitlab-url', help='GitLab URL') parser.add_argument('--gitea-url', help='Gitea URL') parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1') +parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true') +parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true') parser.add_argument('--quiet', help='Enable quiet mode', action='store_true') parser.add_argument('--debug', help='Enable debug mode', action='store_true') +parser.add_argument('--trace', help='Enable trace mode', action='store_true') parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true') parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log') parser.add_argument('--append-log', help='Append log file', action='store_true') @@ -173,12 +210,21 @@ if args.gitea_url: if args.gitea_api_version: GITEA_API_VERSION = args.gitea_api_version +if args.create_missing_groups: + CREATE_MISSING_GROUPS = True + +if args.create_missing_users: + CREATE_MISSING_USERS = True + if args.quiet: QUIET = True if args.debug: DEBUG = True +if args.trace: + TRACE = True + if args.dry_run: DRY_RUN = True @@ -195,8 +241,16 @@ GITEA_URL = GITEA_URL.rstrip('/') # Internal functions +def _trace(message): + if TRACE: + print('\033[1m\033[36m[TRC]\033[0m', message) + + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[TRC] {message}\n') + def _debug(message): - if DEBUG: + if TRACE or DEBUG: print('\033[1m\033[34m[DBG]\033[0m', message) if LOG_FILE: @@ -236,6 +290,8 @@ def check_gitlab(): 'Authorization': f'Bearer {GITLAB_TOKEN}' }) + _trace(f'RESPONSE: {response.json()}') + if response.status_code != 200: response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' raise Exception(f'GitLab endpoint test failed: {response_message}') @@ -251,6 +307,8 @@ def check_gitea(): 'Accept': 'application/json', 'Authorization': f'token {GITEA_TOKEN}' }) + + _trace(f'RESPONSE: {response.json()}') if response.status_code != 200: response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' @@ -260,7 +318,30 @@ def check_gitea(): # Endpoint: GET /api/{GITLAB_API_VERSION}/groups def get_gitlab_groups() -> list: - pass + groups = [] + + _debug(f'REQUEST: {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups') + + response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups', + params={ + 'all_available': 1, + 'per_page': 100, + 'page': 1, + 'top_level_only': 1 + }, + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'Bearer {GITLAB_TOKEN}' + }) + + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'Failed to get GitLab groups: {response_message}') + else: + groups = response.json() + + return groups # Endpoint: GET /api/{GITLAB_API_VERSION}/users def get_gitlab_users() -> list: @@ -272,7 +353,24 @@ def get_gitlab_projects() -> list: # Endpoint: GET /api/{GITEA_API_VERSION}/orgs def get_gitea_groups() -> list: - pass + groups = [] + + _debug(f'REQUEST: {GITEA_URL}/api/{GITEA_API_VERSION}/orgs?all_available=1&per_page=100') + + response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs', + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'token {GITEA_TOKEN}' + }) + + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'Failed to get Gitea groups: {response_message}') + else: + groups = response.json() + + return groups # Endpoint: GET /api/{GITEA_API_VERSION}/users def get_gitea_users() -> list: @@ -283,7 +381,110 @@ def get_gitea_projects() -> list: pass def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict: - pass + + # 0 = exists on both + # 1 = exists on GitLab only + # 2 = exists on Gitea only + # planned structure: + # { + # 'groupname' : 0, + # 'groupname2' : 1, + # 'groupname3' : 2 + # } + + # Example for GitLab groups + # [ + # { + # "id": 15, + # "web_url": "https://git.madaboutpandas.com/groups/administration", + # "name": "Administration", + # "path": "administration", + # "description": "", + # "visibility": "private", + # "share_with_group_lock": false, + # "require_two_factor_authentication": false, + # "two_factor_grace_period": 48, + # "project_creation_level": "developer", + # "auto_devops_enabled": null, + # "subgroup_creation_level": "maintainer", + # "emails_disabled": false, + # "emails_enabled": true, + # "mentions_disabled": null, + # "lfs_enabled": true, + # "math_rendering_limits_enabled": true, + # "lock_math_rendering_limits_enabled": false, + # "default_branch": null, + # "default_branch_protection": 1, + # "default_branch_protection_defaults": { + # "allowed_to_push": [{"access_level": 30}], + # "allow_force_push": false, + # "allowed_to_merge": [{"access_level": 40}] + # }, + # "avatar_url": null, + # "request_access_enabled": true, + # "full_name": "Administration", + # "full_path": "administration", + # "created_at": "2020-08-10T20:27:23.487Z", + # "parent_id": null, + # "organization_id": 1, + # "shared_runners_setting": "enabled", + # "ldap_cn": null, + # "ldap_access": null, + # "wiki_access_level": "enabled" + # } + # ] + + # Example for Gitea groups + # [ + # { + # "id": 3, + # "name": "YerbaBuena", + # "full_name": "", + # "email": "", + # "avatar_url": "http://10.17.1.21/avatars/9f8ea65601abbf666adcec2b128180e4", + # "description": "", + # "website": "", + # "location": "", + # "visibility": "public", + # "repo_admin_change_team_access": True, + # "username": "YerbaBuena", + # } + # ] + + compare_result = {} + missing_matches = 0 + + for gitlab_group in gitlab_groups: + name = gitlab_group['path'] + exists = False + + for gitea_group in gitea_groups: + if name == gitea_group['name']: + exists = True + break + + if exists: + _info(f'GITLAB: Group "{name}" exists on both GitLab and Gitea') + else: + _warn(f'GITLAB: Group "{name}" exists on GitLab only') + missing_matches += 1 + + compare_result[name] = 0 if exists else 1 + + for gitea_group in gitea_groups: + name = gitea_group['name'] + exists = False + + for gitlab_group in gitlab_groups: + if name == gitlab_group['path']: + exists = True + break + + if not exists: + _warn(f'GITEA: Group "{name}" exists on Gitea only') + compare_result[name] = 2 + + return compare_result, missing_matches def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict: pass @@ -291,6 +492,9 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict: def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict: pass +def create_missing_groups(gitlab_groups: list, gitea_groups: list): + pass + def run_migration(): _info('Migrating GitLab groups...') @@ -298,7 +502,20 @@ def run_migration(): gitlab_groups = get_gitlab_groups() gitea_groups = get_gitea_groups() - cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) + _debug(f'Groups on GitLab: {len(gitlab_groups)}') + _trace(f'Groups on GitLab: {gitlab_groups}') + _debug(f'Groups on Gitea: {len(gitea_groups)}') + _trace(f'Groups on Gitea: {gitea_groups}') + + group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) + + if missing_matches > 0: + _warn(f'{missing_matches} groups are missing on Gitea!') + + if missing_matches > 0 and CREATE_MISSING_GROUPS: + _info('Creating missing groups on Gitea...') + + create_missing_groups(gitlab_groups, gitea_groups) _info('Migrating GitLab users...') @@ -314,12 +531,17 @@ def run_migration(): cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects) - - def main(): - if DRY_RUN: - _warn('Running in dry-run mode!') - + _info('Gitlab2Gitea v1.0 - by Zion Networks') + _info('------------------------------------') + _info('') + + if sys.version_info < (3, 6): + _error('Python 3.6 or higher is required!') + return + else: + _debug(f'Python version: {sys.version_info.major}.{sys.version_info.minor}') + if not GITLAB_TOKEN: _error('GitLab access token is missing!') return @@ -331,11 +553,14 @@ def main(): if LOG_FILE and not APPEND_LOG: with open(LOG_FILE, 'w') as log_file: log_file.write('') - + _debug(f'GitLab URL: {GITLAB_URL}') _debug(f'Gitea URL: {GITEA_URL}') _debug(f'Logging to: {LOG_FILE}') + if DRY_RUN: + _warn('Running in dry-run mode!') + _info('Testing endpoints...') try: @@ -353,4 +578,4 @@ def main(): _error(f'An error occurred: {e}') if __name__ == '__main__': - main() \ No newline at end of file + main()