ZionNetworks/linux-bash-scripts
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: --create-missing-projects; started working on user migration; blueprinted further migration functions

Browse Source
This commit is contained in:
Enrico Ludwig 2024-07-14 13:02:08 +02:00
parent b4eacc471b
commit 6b1b522f20
1 changed files with 243 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

254
gitlab2gitea/gitlab2gitea.py
View File

@ -15,8 +15,9 @@
# --gitea-url <GITEA_URL> Gitea URL
# --gitea-api-version <GITEA_API_VERSION> Gitea API version (default: v1)
#
# --create-missing-groups Create missing groups on Gitea (default: False)
# --create-missing-users Create missing users on Gitea (default: False)
# --create-missing-groups Create missing groups on Gitea (default: True)
# --create-missing-users Create missing users on Gitea (default: True)
# --create-missing-projects Create missing projects on Gitea (default: True)
#
# --quiet Enable quiet mode (default: False)
# --debug Enable debug mode (default: False)
@ -47,8 +48,9 @@ GITEA_API_VERSION = 'v1'
# Settings - General Repository
CREATE_MISSING_GROUPS = False
CREATE_MISSING_USERS = False
CREATE_MISSING_GROUPS = True
CREATE_MISSING_USERS = True
CREATE_MISSING_PROJECTS = True
# Settings - General
@ -65,6 +67,7 @@ import os
import sys
import argparse
import requests
import traceback
# Set cwd to script directory
@ -95,6 +98,9 @@ if 'CREATE_MISSING_GROUPS' in os.environ:
if 'CREATE_MISSING_USERS' in os.environ:
CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS'])
if 'CREATE_MISSING_PROJECTS' in os.environ:
CREATE_MISSING_PROJECTS = bool(os.environ['CREATE_MISSING_PROJECTS'])
if 'QUIET' in os.environ:
QUIET = bool(os.environ['QUIET'])
@ -150,6 +156,10 @@ if os.path.exists('.env'):
if value.lower() == 'true' or value == '1':
CREATE_MISSING_USERS = True
if key == 'CREATE_MISSING_PROJECTS':
if value.lower() == 'true' or value == '1':
CREATE_MISSING_PROJECTS = True
if key == 'QUIET':
QUIET = bool(value)
@ -183,6 +193,7 @@ parser.add_argument('--gitea-url', help='Gitea URL')
parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true')
parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true')
parser.add_argument('--create-missing-projects', help='Create missing projects on Gitea', action='store_true')
parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
parser.add_argument('--debug', help='Enable debug mode', action='store_true')
parser.add_argument('--trace', help='Enable trace mode', action='store_true')
@ -216,6 +227,9 @@ if args.create_missing_groups:
if args.create_missing_users:
CREATE_MISSING_USERS = True
if args.create_missing_projects:
CREATE_MISSING_PROJECTS = True
if args.quiet:
QUIET = True
@ -278,6 +292,27 @@ def _error(message):
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[ERR] {message}\n')
def _exception(exception, custom_message=None):
exc_type, exc_obj, exc_tb = sys.exc_info()
filename = exc_tb.tb_frame.f_code.co_filename
lineno = exc_tb.tb_lineno
formatted_traceback = traceback.format_exc()
# Prepare the exception message
exception_message = (f'{custom_message}\n' if custom_message else '') + (
f'\033[1m\033[31m[EXC]\033[0m {exception} '
f'(file: {filename}, line: {lineno})\n'
f'{formatted_traceback}\n'
)
# Print the exception message to the console
print(exception_message)
# Write the exception message to the log file if defined
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[EXC] {exception_message}\n')
# PROGRAM
def gitlab2gitea_visibility(visibility: str) -> str:
@ -668,10 +703,146 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
return compare_result, missing_matches
def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
pass
# 0 = exists on both
# 1 = exists on GitLab only
# 2 = exists on Gitea only
# planned structure:
# {
# 'username' : 0,
# 'username2' : 1,
# 'username3' : 2
# }
# Structure on Gitlab
# [
# {
# "id": 3,
# "username": "user.name",
# "name": "user.name",
# "state": "active",
# "locked": false,
# "avatar_url": "https://git.example.com/uploads/-/system/user/avatar/3/avatar.png",
# "web_url": "https://git.example.com/user.name",
# "created_at": "2020-07-16T14:43:36.801Z",
# "bio": "",
# "location": "",
# "public_email": "",
# "skype": "",
# "linkedin": "",
# "twitter": "",
# "discord": "",
# "website_url": "",
# "organization": "",
# "job_title": "",
# "pronouns": null,
# "bot": false,
# "work_information": null,
# "local_time": "10:25 AM",
# "last_sign_in_at": "2024-06-30T20:05:29.531Z",
# "confirmed_at": "2020-07-16T14:43:36.692Z",
# "last_activity_on": "2024-07-14",
# "email": "user.name@example.com",
# "theme_id": 11,
# "color_scheme_id": 2,
# "projects_limit": 100000,
# "current_sign_in_at": "2024-07-13T09:14:00.499Z",
# "identities": [
# {
# "provider": "ldapmain",
# "extern_uid": "cn=user.name,ou=internal,ou=users,ou=org,dc=example,dc=com",
# "saml_provider_id": null
# }
# ],
# "can_create_group": true,
# "can_create_project": true,
# "two_factor_enabled": true,
# "external": false,
# "private_profile": false,
# "commit_email": "user.name@example.com",
# "shared_runners_minutes_limit": null,
# "extra_shared_runners_minutes_limit": null,
# "scim_identities": [],
# "is_admin": true,
# "note": "",
# "namespace_id": 3,
# "created_by": null,
# "email_reset_offered_at": null,
# "using_license_seat": false
# }
# ]
# Structure on Gitea
# {
# "data": [
# {
# "id": 1,
# "login": "user.name",
# "login_name": "",
# "source_id": 0,
# "full_name": "",
# "email": "user.name@example.com",
# "avatar_url": "http://git.example.com/avatars/a81823ace1c9fa7ab59a61ca6e2c34b0",
# "html_url": "http://git.example.com/user.name",
# "language": "en-US",
# "is_admin": true,
# "last_login": "1970-01-01T01:00:00+01:00",
# "created": "2024-07-13T11:04:57+02:00",
# "restricted": false,
# "active": true,
# "prohibit_login": false,
# "location": "",
# "website": "",
# "description": "",
# "visibility": "private",
# "followers_count": 0,
# "following_count": 0,
# "starred_repos_count": 0,
# "username": "user.name"
# }
# ],
# "ok": true
# }
compare_result = {}
missing_matches = 0
for gitlab_user in gitlab_users:
name = gitlab_user['username']
exists = False
for gitea_user in gitea_users:
if name == gitea_user['login']:
exists = True
break
if exists:
_info(f'GITLAB: User "{name}" exists on both GitLab and Gitea')
else:
_warn(f'GITLAB: User "{name}" exists on GitLab only')
missing_matches += 1
compare_result[name] = 0 if exists else 1
for gitea_user in gitea_users:
name = gitea_user['login']
exists = False
for gitlab_user in gitlab_users:
if name == gitlab_user['username']:
exists = True
break
if not exists:
_warn(f'GITEA: User "{name}" exists on Gitea only')
compare_result[name] = 2
return compare_result, missing_matches
def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
pass
return {}, 0
def create_missing_groups(gitlab_groups: list, gitea_groups: list):
@ -690,7 +861,13 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list):
try:
migrate_gitlab_group_to_gitea(gitlab_group)
except Exception as e:
_error(f'Failed to create Gitea group "{name}": {e}')
_exception(f'Failed to create Gitea group "{name}": {e}', e)
def create_missing_users(gitlab_users: list, gitea_users: list):
pass
def create_missing_projects(gitlab_projects: list, gitea_projects: list):
pass
def update_existing_groups(gitlab_groups: list, gitea_groups: list):
for gitlab_group in gitlab_groups:
@ -713,7 +890,10 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list):
'full_name': gitlab_group['full_name']
})
except Exception as e:
_error(f'Failed to update Gitea group "{name}": {e}')
_exception(f'Failed to update Gitea group "{name}": {e}', e)
def update_existing_users(gitlab_users: list, gitea_users: list):
pass
def migrate_groups():
gitlab_groups = get_gitlab_groups()
@ -724,7 +904,11 @@ def migrate_groups():
_info(f'Groups on Gitea: {len(gitea_groups)}')
_trace(f'Groups on Gitea: {gitea_groups}')
try:
group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
except Exception as e:
_exception(f'Failed to compare GitLab and Gitea groups: {e}', e)
return
if missing_matches > 0:
_warn(f'{missing_matches} groups are missing on Gitea!')
@ -733,7 +917,10 @@ def migrate_groups():
_info('Creating missing groups on Gitea...')
if not DRY_RUN:
try:
create_missing_groups(gitlab_groups, gitea_groups)
except Exception as e:
_exception(f'Failed to create missing groups: {e}', e)
else:
_warn('Dry-run mode enabled, skipping creation of missing groups on Gitea...')
@ -745,7 +932,7 @@ def migrate_groups():
else:
_warn('Dry-run mode enabled, skipping update of existing groups on Gitea...')
except Exception as e:
_error(f'Failed to update existing groups: {e}')
_exception(f'Failed to update existing groups: {e}', e)
def migrate_users():
gitlab_users = get_gitlab_users()
@ -756,7 +943,33 @@ def migrate_users():
_info(f'Users on Gitea: {len(gitea_users)}')
_trace(f'Users on Gitea: {gitea_users}')
cmp_gitlab_gitea_users(gitlab_users, gitea_users)
try:
user_result, missing_matches = cmp_gitlab_gitea_users(gitlab_users, gitea_users['data'])
except Exception as e:
_exception(f'Failed to compare GitLab and Gitea users: {e}', e)
return
if missing_matches > 0:
_warn(f'{missing_matches} users are missing on Gitea!')
if missing_matches > 0 and CREATE_MISSING_USERS:
_info('Creating missing users on Gitea...')
if not DRY_RUN:
create_missing_users(gitlab_users, gitea_users)
else:
_warn('Dry-run mode enabled, skipping creation of missing users on Gitea...')
_info('Updating existing users on Gitea...')
try:
if not DRY_RUN:
update_existing_users(gitlab_users, gitea_users)
else:
_warn('Dry-run mode enabled, skipping update of existing users on Gitea...')
except Exception as e:
_exception(f'Failed to update existing users: {e}', e)
def migrate_projects():
gitlab_projects = get_gitlab_projects()
@ -767,7 +980,22 @@ def migrate_projects():
_info(f'Projects on Gitea: {len(gitea_projects)}')
_trace(f'Projects on Gitea: {gitea_projects}')
cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
try:
project_result, missing_matches = cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
except Exception as e:
_exception(f'Failed to compare GitLab and Gitea projects: {e}', e)
return
if missing_matches > 0:
_warn(f'{missing_matches} projects are missing on Gitea!')
if missing_matches > 0 and CREATE_MISSING_PROJECTS:
_info('Creating missing projects on Gitea...')
if not DRY_RUN:
create_missing_projects(gitlab_projects, gitea_projects)
else:
_warn('Dry-run mode enabled, skipping creation of missing projects on Gitea...')
def run_migration():
@ -816,7 +1044,7 @@ def main():
check_gitlab()
check_gitea()
except Exception as e:
_error(f'An error occurred: {e}')
_exception(f'An error occurred: {e}', e)
return
_info('Starting migration...')
@ -824,7 +1052,7 @@ def main():
try:
run_migration()
except Exception as e:
_error(f'An error occurred: {e}')
_exception(f'An error occurred: {e}', e)
if __name__ == '__main__':
main()