From 6b1b522f203ae3554ff518e6f058934129361f55 Mon Sep 17 00:00:00 2001 From: Enrico Ludwig Date: Sun, 14 Jul 2024 13:02:08 +0200 Subject: [PATCH] add: --create-missing-projects; started working on user migration; blueprinted further migration functions --- gitlab2gitea/gitlab2gitea.py | 258 +++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 15 deletions(-) diff --git a/gitlab2gitea/gitlab2gitea.py b/gitlab2gitea/gitlab2gitea.py index 1fd2c2f..2dd966b 100644 --- a/gitlab2gitea/gitlab2gitea.py +++ b/gitlab2gitea/gitlab2gitea.py @@ -15,8 +15,9 @@ # --gitea-url Gitea URL # --gitea-api-version Gitea API version (default: v1) # -# --create-missing-groups Create missing groups on Gitea (default: False) -# --create-missing-users Create missing users on Gitea (default: False) +# --create-missing-groups Create missing groups on Gitea (default: True) +# --create-missing-users Create missing users on Gitea (default: True) +# --create-missing-projects Create missing projects on Gitea (default: True) # # --quiet Enable quiet mode (default: False) # --debug Enable debug mode (default: False) @@ -47,8 +48,9 @@ GITEA_API_VERSION = 'v1' # Settings - General Repository -CREATE_MISSING_GROUPS = False -CREATE_MISSING_USERS = False +CREATE_MISSING_GROUPS = True +CREATE_MISSING_USERS = True +CREATE_MISSING_PROJECTS = True # Settings - General @@ -65,6 +67,7 @@ import os import sys import argparse import requests +import traceback # Set cwd to script directory @@ -95,6 +98,9 @@ if 'CREATE_MISSING_GROUPS' in os.environ: if 'CREATE_MISSING_USERS' in os.environ: CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS']) +if 'CREATE_MISSING_PROJECTS' in os.environ: + CREATE_MISSING_PROJECTS = bool(os.environ['CREATE_MISSING_PROJECTS']) + if 'QUIET' in os.environ: QUIET = bool(os.environ['QUIET']) @@ -150,6 +156,10 @@ if os.path.exists('.env'): if value.lower() == 'true' or value == '1': CREATE_MISSING_USERS = True + if key == 'CREATE_MISSING_PROJECTS': + if value.lower() == 'true' or value == '1': + CREATE_MISSING_PROJECTS = True + if key == 'QUIET': QUIET = bool(value) @@ -183,6 +193,7 @@ parser.add_argument('--gitea-url', help='Gitea URL') parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1') parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true') parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true') +parser.add_argument('--create-missing-projects', help='Create missing projects on Gitea', action='store_true') parser.add_argument('--quiet', help='Enable quiet mode', action='store_true') parser.add_argument('--debug', help='Enable debug mode', action='store_true') parser.add_argument('--trace', help='Enable trace mode', action='store_true') @@ -216,6 +227,9 @@ if args.create_missing_groups: if args.create_missing_users: CREATE_MISSING_USERS = True +if args.create_missing_projects: + CREATE_MISSING_PROJECTS = True + if args.quiet: QUIET = True @@ -278,6 +292,27 @@ def _error(message): with open(LOG_FILE, 'a') as log_file: log_file.write(f'[ERR] {message}\n') +def _exception(exception, custom_message=None): + exc_type, exc_obj, exc_tb = sys.exc_info() + filename = exc_tb.tb_frame.f_code.co_filename + lineno = exc_tb.tb_lineno + formatted_traceback = traceback.format_exc() + + # Prepare the exception message + exception_message = (f'{custom_message}\n' if custom_message else '') + ( + f'\033[1m\033[31m[EXC]\033[0m {exception} ' + f'(file: {filename}, line: {lineno})\n' + f'{formatted_traceback}\n' + ) + + # Print the exception message to the console + print(exception_message) + + # Write the exception message to the log file if defined + if LOG_FILE: + with open(LOG_FILE, 'a') as log_file: + log_file.write(f'[EXC] {exception_message}\n') + # PROGRAM def gitlab2gitea_visibility(visibility: str) -> str: @@ -668,10 +703,146 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict: return compare_result, missing_matches def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict: - pass + + # 0 = exists on both + # 1 = exists on GitLab only + # 2 = exists on Gitea only + # planned structure: + # { + # 'username' : 0, + # 'username2' : 1, + # 'username3' : 2 + # } + + # Structure on Gitlab + # [ + # { + # "id": 3, + # "username": "user.name", + # "name": "user.name", + # "state": "active", + # "locked": false, + # "avatar_url": "https://git.example.com/uploads/-/system/user/avatar/3/avatar.png", + # "web_url": "https://git.example.com/user.name", + # "created_at": "2020-07-16T14:43:36.801Z", + # "bio": "", + # "location": "", + # "public_email": "", + # "skype": "", + # "linkedin": "", + # "twitter": "", + # "discord": "", + # "website_url": "", + # "organization": "", + # "job_title": "", + # "pronouns": null, + # "bot": false, + # "work_information": null, + # "local_time": "10:25 AM", + # "last_sign_in_at": "2024-06-30T20:05:29.531Z", + # "confirmed_at": "2020-07-16T14:43:36.692Z", + # "last_activity_on": "2024-07-14", + # "email": "user.name@example.com", + # "theme_id": 11, + # "color_scheme_id": 2, + # "projects_limit": 100000, + # "current_sign_in_at": "2024-07-13T09:14:00.499Z", + # "identities": [ + # { + # "provider": "ldapmain", + # "extern_uid": "cn=user.name,ou=internal,ou=users,ou=org,dc=example,dc=com", + # "saml_provider_id": null + # } + # ], + # "can_create_group": true, + # "can_create_project": true, + # "two_factor_enabled": true, + # "external": false, + # "private_profile": false, + # "commit_email": "user.name@example.com", + # "shared_runners_minutes_limit": null, + # "extra_shared_runners_minutes_limit": null, + # "scim_identities": [], + # "is_admin": true, + # "note": "", + # "namespace_id": 3, + # "created_by": null, + # "email_reset_offered_at": null, + # "using_license_seat": false + # } + # ] + + # Structure on Gitea + # { + # "data": [ + # { + # "id": 1, + # "login": "user.name", + # "login_name": "", + # "source_id": 0, + # "full_name": "", + # "email": "user.name@example.com", + # "avatar_url": "http://git.example.com/avatars/a81823ace1c9fa7ab59a61ca6e2c34b0", + # "html_url": "http://git.example.com/user.name", + # "language": "en-US", + # "is_admin": true, + # "last_login": "1970-01-01T01:00:00+01:00", + # "created": "2024-07-13T11:04:57+02:00", + # "restricted": false, + # "active": true, + # "prohibit_login": false, + # "location": "", + # "website": "", + # "description": "", + # "visibility": "private", + # "followers_count": 0, + # "following_count": 0, + # "starred_repos_count": 0, + # "username": "user.name" + # } + # ], + # "ok": true + # } + + compare_result = {} + missing_matches = 0 + + for gitlab_user in gitlab_users: + name = gitlab_user['username'] + exists = False + + for gitea_user in gitea_users: + if name == gitea_user['login']: + exists = True + break + + if exists: + _info(f'GITLAB: User "{name}" exists on both GitLab and Gitea') + else: + _warn(f'GITLAB: User "{name}" exists on GitLab only') + missing_matches += 1 + + compare_result[name] = 0 if exists else 1 + + for gitea_user in gitea_users: + name = gitea_user['login'] + exists = False + + for gitlab_user in gitlab_users: + if name == gitlab_user['username']: + exists = True + break + + if not exists: + _warn(f'GITEA: User "{name}" exists on Gitea only') + compare_result[name] = 2 + + return compare_result, missing_matches + def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict: - pass + + return {}, 0 def create_missing_groups(gitlab_groups: list, gitea_groups: list): @@ -690,7 +861,13 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list): try: migrate_gitlab_group_to_gitea(gitlab_group) except Exception as e: - _error(f'Failed to create Gitea group "{name}": {e}') + _exception(f'Failed to create Gitea group "{name}": {e}', e) + +def create_missing_users(gitlab_users: list, gitea_users: list): + pass + +def create_missing_projects(gitlab_projects: list, gitea_projects: list): + pass def update_existing_groups(gitlab_groups: list, gitea_groups: list): for gitlab_group in gitlab_groups: @@ -713,7 +890,10 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list): 'full_name': gitlab_group['full_name'] }) except Exception as e: - _error(f'Failed to update Gitea group "{name}": {e}') + _exception(f'Failed to update Gitea group "{name}": {e}', e) + +def update_existing_users(gitlab_users: list, gitea_users: list): + pass def migrate_groups(): gitlab_groups = get_gitlab_groups() @@ -724,7 +904,11 @@ def migrate_groups(): _info(f'Groups on Gitea: {len(gitea_groups)}') _trace(f'Groups on Gitea: {gitea_groups}') - group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) + try: + group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) + except Exception as e: + _exception(f'Failed to compare GitLab and Gitea groups: {e}', e) + return if missing_matches > 0: _warn(f'{missing_matches} groups are missing on Gitea!') @@ -733,7 +917,10 @@ def migrate_groups(): _info('Creating missing groups on Gitea...') if not DRY_RUN: - create_missing_groups(gitlab_groups, gitea_groups) + try: + create_missing_groups(gitlab_groups, gitea_groups) + except Exception as e: + _exception(f'Failed to create missing groups: {e}', e) else: _warn('Dry-run mode enabled, skipping creation of missing groups on Gitea...') @@ -745,7 +932,7 @@ def migrate_groups(): else: _warn('Dry-run mode enabled, skipping update of existing groups on Gitea...') except Exception as e: - _error(f'Failed to update existing groups: {e}') + _exception(f'Failed to update existing groups: {e}', e) def migrate_users(): gitlab_users = get_gitlab_users() @@ -756,7 +943,33 @@ def migrate_users(): _info(f'Users on Gitea: {len(gitea_users)}') _trace(f'Users on Gitea: {gitea_users}') - cmp_gitlab_gitea_users(gitlab_users, gitea_users) + try: + user_result, missing_matches = cmp_gitlab_gitea_users(gitlab_users, gitea_users['data']) + except Exception as e: + _exception(f'Failed to compare GitLab and Gitea users: {e}', e) + return + + if missing_matches > 0: + _warn(f'{missing_matches} users are missing on Gitea!') + + if missing_matches > 0 and CREATE_MISSING_USERS: + _info('Creating missing users on Gitea...') + + if not DRY_RUN: + create_missing_users(gitlab_users, gitea_users) + else: + _warn('Dry-run mode enabled, skipping creation of missing users on Gitea...') + + _info('Updating existing users on Gitea...') + + try: + if not DRY_RUN: + update_existing_users(gitlab_users, gitea_users) + else: + _warn('Dry-run mode enabled, skipping update of existing users on Gitea...') + except Exception as e: + _exception(f'Failed to update existing users: {e}', e) + def migrate_projects(): gitlab_projects = get_gitlab_projects() @@ -767,7 +980,22 @@ def migrate_projects(): _info(f'Projects on Gitea: {len(gitea_projects)}') _trace(f'Projects on Gitea: {gitea_projects}') - cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects) + try: + project_result, missing_matches = cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects) + except Exception as e: + _exception(f'Failed to compare GitLab and Gitea projects: {e}', e) + return + + if missing_matches > 0: + _warn(f'{missing_matches} projects are missing on Gitea!') + + if missing_matches > 0 and CREATE_MISSING_PROJECTS: + _info('Creating missing projects on Gitea...') + + if not DRY_RUN: + create_missing_projects(gitlab_projects, gitea_projects) + else: + _warn('Dry-run mode enabled, skipping creation of missing projects on Gitea...') def run_migration(): @@ -816,7 +1044,7 @@ def main(): check_gitlab() check_gitea() except Exception as e: - _error(f'An error occurred: {e}') + _exception(f'An error occurred: {e}', e) return _info('Starting migration...') @@ -824,7 +1052,7 @@ def main(): try: run_migration() except Exception as e: - _error(f'An error occurred: {e}') + _exception(f'An error occurred: {e}', e) if __name__ == '__main__': main()