ZionNetworks/linux-bash-scripts
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Further work on gitlab2gitea

Browse Source
This commit is contained in:
Enrico Ludwig 2024-07-13 17:15:10 +02:00
parent f8e655c62f
commit 0c1e91d656
1 changed files with 239 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

253
gitlab2gitea/gitlab2gitea.py
View File

@ -15,8 +15,12 @@
# --gitea-url <GITEA_URL> Gitea URL
# --gitea-api-version <GITEA_API_VERSION> Gitea API version (default: v1)
#
# --create-missing-groups Create missing groups on Gitea (default: False)
# --create-missing-users Create missing users on Gitea (default: False)
#
# --quiet Enable quiet mode (default: False)
# --debug Enable debug mode (default: False)
# --trace Enable trace mode (default: False)
# --dry-run Enable dry-run mode (default: False)
# --log-file <LOG_FILE> Log file (default: gitlab2gitea.log)
# If not set, logs will be only printed to stdout
@ -41,9 +45,15 @@ GITEA_URL = ''
GITEA_TOKEN = ''
GITEA_API_VERSION = 'v1'
# Settings - General Repository
CREATE_MISSING_GROUPS = False
CREATE_MISSING_USERS = False
# Settings - General
DEBUG = False
TRACE = False
DRY_RUN = False
LOG_FILE = 'gitlab2gitea.log'
APPEND_LOG = False
@ -52,6 +62,7 @@ QUIET = False
# Imports
import os
import sys
import argparse
import requests
@ -78,12 +89,21 @@ if 'GITEA_TOKEN' in os.environ:
if 'GITEA_API_VERSION' in os.environ:
GITEA_API_VERSION = os.environ['GITEA_API_VERSION']
if 'CREATE_MISSING_GROUPS' in os.environ:
CREATE_MISSING_GROUPS = bool(os.environ['CREATE_MISSING_GROUPS'])
if 'CREATE_MISSING_USERS' in os.environ:
CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS'])
if 'QUIET' in os.environ:
QUIET = bool(os.environ['QUIET'])
if 'DEBUG' in os.environ:
DEBUG = bool(os.environ['DEBUG'])
if 'TRACE' in os.environ:
TRACE = bool(os.environ['TRACE'])
if 'DRY_RUN' in os.environ:
DRY_RUN = bool(os.environ['DRY_RUN'])
@ -121,15 +141,29 @@ if os.path.exists('.env'):
if key == 'GITEA_API_VERSION':
GITEA_API_VERSION = value
if key == 'CREATE_MISSING_GROUPS':
if value.lower() == 'true' or value == '1':
CREATE_MISSING_GROUPS = True
if key == 'CREATE_MISSING_USERS':
if value.lower() == 'true' or value == '1':
CREATE_MISSING_USERS = True
if key == 'QUIET':
QUIET = bool(value)
if key == 'DEBUG':
DEBUG = bool(value)
if value.lower() == 'true' or value == '1':
DEBUG = True
if key == 'TRACE':
if value.lower() == 'true' or value == '1':
TRACE = True
if key == 'DRY_RUN':
DRY_RUN = bool(value)
if value.lower() == 'true' or value == '1':
DRY_RUN = True
if key == 'LOG_FILE':
LOG_FILE = value
@ -147,8 +181,11 @@ parser.add_argument('--gitea-token', help='Gitea access token')
parser.add_argument('--gitlab-url', help='GitLab URL')
parser.add_argument('--gitea-url', help='Gitea URL')
parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true')
parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true')
parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
parser.add_argument('--debug', help='Enable debug mode', action='store_true')
parser.add_argument('--trace', help='Enable trace mode', action='store_true')
parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true')
parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log')
parser.add_argument('--append-log', help='Append log file', action='store_true')
@ -173,12 +210,21 @@ if args.gitea_url:
if args.gitea_api_version:
GITEA_API_VERSION = args.gitea_api_version
if args.create_missing_groups:
CREATE_MISSING_GROUPS = True
if args.create_missing_users:
CREATE_MISSING_USERS = True
if args.quiet:
QUIET = True
if args.debug:
DEBUG = True
if args.trace:
TRACE = True
if args.dry_run:
DRY_RUN = True
@ -195,8 +241,16 @@ GITEA_URL = GITEA_URL.rstrip('/')
# Internal functions
def _trace(message):
if TRACE:
print('\033[1m\033[36m[TRC]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[TRC] {message}\n')
def _debug(message):
if DEBUG:
if TRACE or DEBUG:
print('\033[1m\033[34m[DBG]\033[0m', message)
if LOG_FILE:
@ -236,6 +290,8 @@ def check_gitlab():
'Authorization': f'Bearer {GITLAB_TOKEN}'
})
_trace(f'RESPONSE: {response.json()}')
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'GitLab endpoint test failed: {response_message}')
@ -251,6 +307,8 @@ def check_gitea():
'Accept': 'application/json',
'Authorization': f'token {GITEA_TOKEN}'
})
_trace(f'RESPONSE: {response.json()}')
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
@ -260,7 +318,30 @@ def check_gitea():
# Endpoint: GET /api/{GITLAB_API_VERSION}/groups
def get_gitlab_groups() -> list:
pass
groups = []
_debug(f'REQUEST: {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups')
response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups',
params={
'all_available': 1,
'per_page': 100,
'page': 1,
'top_level_only': 1
},
headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {GITLAB_TOKEN}'
})
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to get GitLab groups: {response_message}')
else:
groups = response.json()
return groups
# Endpoint: GET /api/{GITLAB_API_VERSION}/users
def get_gitlab_users() -> list:
@ -272,7 +353,24 @@ def get_gitlab_projects() -> list:
# Endpoint: GET /api/{GITEA_API_VERSION}/orgs
def get_gitea_groups() -> list:
pass
groups = []
_debug(f'REQUEST: {GITEA_URL}/api/{GITEA_API_VERSION}/orgs?all_available=1&per_page=100')
response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs',
headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'token {GITEA_TOKEN}'
})
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to get Gitea groups: {response_message}')
else:
groups = response.json()
return groups
# Endpoint: GET /api/{GITEA_API_VERSION}/users
def get_gitea_users() -> list:
@ -283,7 +381,110 @@ def get_gitea_projects() -> list:
pass
def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
pass
# 0 = exists on both
# 1 = exists on GitLab only
# 2 = exists on Gitea only
# planned structure:
# {
# 'groupname' : 0,
# 'groupname2' : 1,
# 'groupname3' : 2
# }
# Example for GitLab groups
# [
# {
# "id": 15,
# "web_url": "https://git.madaboutpandas.com/groups/administration",
# "name": "Administration",
# "path": "administration",
# "description": "",
# "visibility": "private",
# "share_with_group_lock": false,
# "require_two_factor_authentication": false,
# "two_factor_grace_period": 48,
# "project_creation_level": "developer",
# "auto_devops_enabled": null,
# "subgroup_creation_level": "maintainer",
# "emails_disabled": false,
# "emails_enabled": true,
# "mentions_disabled": null,
# "lfs_enabled": true,
# "math_rendering_limits_enabled": true,
# "lock_math_rendering_limits_enabled": false,
# "default_branch": null,
# "default_branch_protection": 1,
# "default_branch_protection_defaults": {
# "allowed_to_push": [{"access_level": 30}],
# "allow_force_push": false,
# "allowed_to_merge": [{"access_level": 40}]
# },
# "avatar_url": null,
# "request_access_enabled": true,
# "full_name": "Administration",
# "full_path": "administration",
# "created_at": "2020-08-10T20:27:23.487Z",
# "parent_id": null,
# "organization_id": 1,
# "shared_runners_setting": "enabled",
# "ldap_cn": null,
# "ldap_access": null,
# "wiki_access_level": "enabled"
# }
# ]
# Example for Gitea groups
# [
# {
# "id": 3,
# "name": "YerbaBuena",
# "full_name": "",
# "email": "",
# "avatar_url": "http://10.17.1.21/avatars/9f8ea65601abbf666adcec2b128180e4",
# "description": "",
# "website": "",
# "location": "",
# "visibility": "public",
# "repo_admin_change_team_access": True,
# "username": "YerbaBuena",
# }
# ]
compare_result = {}
missing_matches = 0
for gitlab_group in gitlab_groups:
name = gitlab_group['path']
exists = False
for gitea_group in gitea_groups:
if name == gitea_group['name']:
exists = True
break
if exists:
_info(f'GITLAB: Group "{name}" exists on both GitLab and Gitea')
else:
_warn(f'GITLAB: Group "{name}" exists on GitLab only')
missing_matches += 1
compare_result[name] = 0 if exists else 1
for gitea_group in gitea_groups:
name = gitea_group['name']
exists = False
for gitlab_group in gitlab_groups:
if name == gitlab_group['path']:
exists = True
break
if not exists:
_warn(f'GITEA: Group "{name}" exists on Gitea only')
compare_result[name] = 2
return compare_result, missing_matches
def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
pass
@ -291,6 +492,9 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
pass
def create_missing_groups(gitlab_groups: list, gitea_groups: list):
pass
def run_migration():
_info('Migrating GitLab groups...')
@ -298,7 +502,20 @@ def run_migration():
gitlab_groups = get_gitlab_groups()
gitea_groups = get_gitea_groups()
cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
_debug(f'Groups on GitLab: {len(gitlab_groups)}')
_trace(f'Groups on GitLab: {gitlab_groups}')
_debug(f'Groups on Gitea: {len(gitea_groups)}')
_trace(f'Groups on Gitea: {gitea_groups}')
group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
if missing_matches > 0:
_warn(f'{missing_matches} groups are missing on Gitea!')
if missing_matches > 0 and CREATE_MISSING_GROUPS:
_info('Creating missing groups on Gitea...')
create_missing_groups(gitlab_groups, gitea_groups)
_info('Migrating GitLab users...')
@ -314,12 +531,17 @@ def run_migration():
cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
def main():
if DRY_RUN:
_warn('Running in dry-run mode!')
_info('Gitlab2Gitea v1.0 - by Zion Networks')
_info('------------------------------------')
_info('')
if sys.version_info < (3, 6):
_error('Python 3.6 or higher is required!')
return
else:
_debug(f'Python version: {sys.version_info.major}.{sys.version_info.minor}')
if not GITLAB_TOKEN:
_error('GitLab access token is missing!')
return
@ -331,11 +553,14 @@ def main():
if LOG_FILE and not APPEND_LOG:
with open(LOG_FILE, 'w') as log_file:
log_file.write('')
_debug(f'GitLab URL: {GITLAB_URL}')
_debug(f'Gitea URL: {GITEA_URL}')
_debug(f'Logging to: {LOG_FILE}')
if DRY_RUN:
_warn('Running in dry-run mode!')
_info('Testing endpoints...')
try:
@ -353,4 +578,4 @@ def main():
_error(f'An error occurred: {e}')
if __name__ == '__main__':
main()
main()