This repository has been archived on 2025-04-03. You can view files and clone it, but cannot push or open issues or pull requests.
linux-bash-scripts/gitlab2gitea/gitlab2gitea.py

657 lines
20 KiB
Python

# Description: Script to migrate repositories from GitLab to Gitea
# Author: Enrico Ludwig <enrico.ludwig@zion-networks.de>
# Version: 1.0
# Date: 2021-01-06
#
# Precedence of settings: arguments > .env file > environment variables > default values
#
# Usage:
# python3 gitlab2gitea.py [options]
#
# --gitlab-token <GITLAB_TOKEN> GitLab access token
# --gitlab-url <GITLAB_URL> GitLab URL
# --gitlab-api-version <GITLAB_API_VERSION> GitLab API version (default: v4)
# --gitea-token <GITEA_TOKEN> Gitea access token
# --gitea-url <GITEA_URL> Gitea URL
# --gitea-api-version <GITEA_API_VERSION> Gitea API version (default: v1)
#
# --create-missing-groups Create missing groups on Gitea (default: False)
# --create-missing-users Create missing users on Gitea (default: False)
#
# --quiet Enable quiet mode (default: False)
# --debug Enable debug mode (default: False)
# --trace Enable trace mode (default: False)
# --dry-run Enable dry-run mode (default: False)
# --log-file <LOG_FILE> Log file (default: gitlab2gitea.log)
# If not set, logs will be only printed to stdout
# --append-log Append log file (default: False)
#
# Example:
# python3 gitlab2gitea.py \
# --gitlab-url <GITLAB_URL> \
# --gitlab-token <GITLAB_TOKEN> \
# --gitea-url <GITEA_URL> \
# --gitea-token <GITEA_TOKEN>
# Settings - GitLab
GITLAB_URL = ''
GITLAB_TOKEN = ''
GITLAB_API_VERSION = 'v4'
# Settings - Gitea
GITEA_URL = ''
GITEA_TOKEN = ''
GITEA_API_VERSION = 'v1'
# Settings - General Repository
CREATE_MISSING_GROUPS = False
CREATE_MISSING_USERS = False
# Settings - General
DEBUG = False
TRACE = False
DRY_RUN = False
LOG_FILE = 'gitlab2gitea.log'
APPEND_LOG = False
QUIET = False
# Imports
import os
import sys
import argparse
import requests
# Set cwd to script directory
os.chdir(os.path.dirname(os.path.realpath(__file__)))
# Read environment variables
if 'GITLAB_URL' in os.environ:
GITLAB_URL = os.environ['GITLAB_URL']
if 'GITLAB_TOKEN' in os.environ:
GITLAB_TOKEN = os.environ['GITLAB_TOKEN']
if 'GITLAB_API_VERSION' in os.environ:
GITLAB_API_VERSION = os.environ['GITLAB_API_VERSION']
if 'GITEA_URL' in os.environ:
GITEA_URL = os.environ['GITEA_URL']
if 'GITEA_TOKEN' in os.environ:
GITEA_TOKEN = os.environ['GITEA_TOKEN']
if 'GITEA_API_VERSION' in os.environ:
GITEA_API_VERSION = os.environ['GITEA_API_VERSION']
if 'CREATE_MISSING_GROUPS' in os.environ:
CREATE_MISSING_GROUPS = bool(os.environ['CREATE_MISSING_GROUPS'])
if 'CREATE_MISSING_USERS' in os.environ:
CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS'])
if 'QUIET' in os.environ:
QUIET = bool(os.environ['QUIET'])
if 'DEBUG' in os.environ:
DEBUG = bool(os.environ['DEBUG'])
if 'TRACE' in os.environ:
TRACE = bool(os.environ['TRACE'])
if 'DRY_RUN' in os.environ:
DRY_RUN = bool(os.environ['DRY_RUN'])
if 'LOG_FILE' in os.environ:
LOG_FILE = os.environ['LOG_FILE']
if 'APPEND_LOG' in os.environ:
APPEND_LOG = bool(os.environ['APPEND_LOG'])
# Read .env file if exists and override environment variables
if os.path.exists('.env'):
with open('.env', 'r') as env_file:
for line in env_file:
# skip comments, empty lines and lines without '='
if line.startswith('#') or not '=' in line:
continue
key, value = line.strip().split('=')
if key == 'GITLAB_API_VERSION':
GITLAB_API_VERSION = value
if key == 'GITLAB_URL':
GITLAB_URL = value
if key == 'GITLAB_TOKEN':
GITLAB_TOKEN = value
if key == 'GITEA_URL':
GITEA_URL = value
if key == 'GITEA_TOKEN':
GITEA_TOKEN = value
if key == 'GITEA_API_VERSION':
GITEA_API_VERSION = value
if key == 'CREATE_MISSING_GROUPS':
if value.lower() == 'true' or value == '1':
CREATE_MISSING_GROUPS = True
if key == 'CREATE_MISSING_USERS':
if value.lower() == 'true' or value == '1':
CREATE_MISSING_USERS = True
if key == 'QUIET':
QUIET = bool(value)
if key == 'DEBUG':
if value.lower() == 'true' or value == '1':
DEBUG = True
if key == 'TRACE':
if value.lower() == 'true' or value == '1':
TRACE = True
if key == 'DRY_RUN':
if value.lower() == 'true' or value == '1':
DRY_RUN = True
if key == 'LOG_FILE':
LOG_FILE = value
if key == 'APPEND_LOG':
APPEND_LOG = bool(value)
# Read arguments and override environment variables
parser = argparse.ArgumentParser(description='Script to migrate repositories from GitLab to Gitea')
parser.add_argument('--gitlab-api-version', help='GitLab API version', default='v4')
parser.add_argument('--gitlab-token', help='GitLab access token')
parser.add_argument('--gitea-token', help='Gitea access token')
parser.add_argument('--gitlab-url', help='GitLab URL')
parser.add_argument('--gitea-url', help='Gitea URL')
parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true')
parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true')
parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
parser.add_argument('--debug', help='Enable debug mode', action='store_true')
parser.add_argument('--trace', help='Enable trace mode', action='store_true')
parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true')
parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log')
parser.add_argument('--append-log', help='Append log file', action='store_true')
args = parser.parse_args()
if args.gitlab_api_version:
GITLAB_API_VERSION = args.gitlab_api_version
if args.gitlab_token:
GITLAB_TOKEN = args.gitlab_token
if args.gitlab_url:
GITLAB_URL = args.gitlab_url
if args.gitea_token:
GITEA_TOKEN = args.gitea_token
if args.gitea_url:
GITEA_URL = args.gitea_url
if args.gitea_api_version:
GITEA_API_VERSION = args.gitea_api_version
if args.create_missing_groups:
CREATE_MISSING_GROUPS = True
if args.create_missing_users:
CREATE_MISSING_USERS = True
if args.quiet:
QUIET = True
if args.debug:
DEBUG = True
if args.trace:
TRACE = True
if args.dry_run:
DRY_RUN = True
if args.log_file:
LOG_FILE = args.log_file
if args.append_log:
APPEND_LOG = True
# Remove trailing slashes from URLs
GITLAB_URL = GITLAB_URL.rstrip('/')
GITEA_URL = GITEA_URL.rstrip('/')
# Internal functions
def _trace(message):
if TRACE:
print('\033[1m\033[36m[TRC]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[TRC] {message}\n')
def _debug(message):
if TRACE or DEBUG:
print('\033[1m\033[34m[DBG]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[DBG] {message}\n')
def _info(message):
print('\033[1m\033[32m[INF]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[INF] {message}\n')
def _warn(message):
print('\033[1m\033[33m[WRN]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[WRN] {message}\n')
def _error(message):
print('\033[1m\033[31m[ERR]\033[0m', message)
if LOG_FILE:
with open(LOG_FILE, 'a') as log_file:
log_file.write(f'[ERR] {message}\n')
# PROGRAM
# Endpoint: GET /api/{GITLAB_API_VERSION}/version
def check_gitlab():
_debug(f'REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/version')
response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/version', headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {GITLAB_TOKEN}'
})
_trace(f'RESPONSE: {response.json()}')
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'GitLab endpoint test failed: {response_message}')
else:
_info(f'GitLab endpoint version: {response.json()["version"]}')
# Endpoint: GET /api/{GITEA_API_VERSION}/version
def check_gitea():
_debug(f'REQUEST: GET {GITEA_URL}/api/{GITEA_API_VERSION}/version')
response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/version', headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'token {GITEA_TOKEN}'
})
_trace(f'RESPONSE: {response.json()}')
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Gitea endpoint test failed: {response_message}')
else:
_info(f'Gitea endpoint version: {response.json()["version"]}')
# Endpoint: GET /api/{GITLAB_API_VERSION}/groups
def get_gitlab_groups() -> list:
groups = []
_debug(f'REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups')
response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups',
params={
'all_available': 1,
'per_page': 100,
'page': 1,
'top_level_only': 1
},
headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {GITLAB_TOKEN}'
})
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to get GitLab groups: {response_message}')
else:
groups = response.json()
return groups
# Endpoint: GET /api/{GITLAB_API_VERSION}/groups/{group_id}
def get_gitlab_group(group_id: int) -> dict:
_debug(f'REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups/{group_id}')
response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups/{group_id}',
headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'Bearer {GITLAB_TOKEN}'
})
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to get GitLab group: {response_message}')
else:
group = response.json()
return group
# Endpoint: GET /api/{GITLAB_API_VERSION}/users
def get_gitlab_users() -> list:
pass
# Endpoint: GET /api/{GITLAB_API_VERSION}/projects
def get_gitlab_projects() -> list:
pass
# Endpoint: GET /api/{GITEA_API_VERSION}/orgs
def get_gitea_groups() -> list:
groups = []
_debug(f'REQUEST: GET {GITEA_URL}/api/{GITEA_API_VERSION}/orgs?all_available=1&per_page=100')
response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs',
headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'token {GITEA_TOKEN}'
})
if response.status_code != 200:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to get Gitea groups: {response_message}')
else:
groups = response.json()
return groups
# Endpoint: GET /api/{GITEA_API_VERSION}/users
def get_gitea_users() -> list:
pass
# Endpoint: GET /api/{GITEA_API_VERSION}/repos
def get_gitea_projects() -> list:
pass
# Endpoint: POST /api/{GITEA_API_VERSION}/orgs
def create_gitea_org(name: str, description: str = '', website: str = '', visibility: str = 'private') -> dict:
gitlab_group = get_gitlab_group(name)
if not gitlab_group:
raise Exception(f'GitLab group "{name}" does not exist!')
# Update detail fields
if description:
gitlab_group['description'] = description
if website:
gitlab_group['web_url'] = website
# Create Gitea group
_debug(f'REQUEST: POST {GITEA_URL}/api/{GITEA_API_VERSION}/orgs')
response = requests.post(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs', json={
'username': name,
'description': gitlab_group['description'],
'website': gitlab_group['web_url'],
'visibility': visibility
}, headers={
'Content-Type': 'application/json',
'Accept': 'application/json',
'Authorization': f'token {GITEA_TOKEN}'
})
_trace(f'RESPONSE: {response.json()}')
if response.status_code != 201:
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
raise Exception(f'Failed to create Gitea group: {response_message}')
else:
group = response.json()
return group
def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
# 0 = exists on both
# 1 = exists on GitLab only
# 2 = exists on Gitea only
# planned structure:
# {
# 'groupname' : 0,
# 'groupname2' : 1,
# 'groupname3' : 2
# }
# Example for GitLab groups
# [
# {
# "id": 15,
# "web_url": "https://gitlab.example.org/groups/developers",
# "name": "Developers",
# "path": "developers",
# "description": "",
# "visibility": "private",
# "share_with_group_lock": false,
# "require_two_factor_authentication": false,
# "two_factor_grace_period": 48,
# "project_creation_level": "developer",
# "auto_devops_enabled": null,
# "subgroup_creation_level": "maintainer",
# "emails_disabled": false,
# "emails_enabled": true,
# "mentions_disabled": null,
# "lfs_enabled": true,
# "math_rendering_limits_enabled": true,
# "lock_math_rendering_limits_enabled": false,
# "default_branch": null,
# "default_branch_protection": 1,
# "default_branch_protection_defaults": {
# "allowed_to_push": [{"access_level": 30}],
# "allow_force_push": false,
# "allowed_to_merge": [{"access_level": 40}]
# },
# "avatar_url": null,
# "request_access_enabled": true,
# "full_name": "Developers",
# "full_path": "developers",
# "created_at": "2020-08-10T20:27:23.487Z",
# "parent_id": null,
# "organization_id": 1,
# "shared_runners_setting": "enabled",
# "ldap_cn": null,
# "ldap_access": null,
# "wiki_access_level": "enabled"
# }
# ]
# Example for Gitea groups
# [
# {
# "id": 3,
# "name": "Developers",
# "full_name": "",
# "email": "",
# "avatar_url": "http://gitea.example.org/avatars/9f8ea65601abbf666adcec2b128180e4",
# "description": "",
# "website": "",
# "location": "",
# "visibility": "public",
# "repo_admin_change_team_access": True,
# "username": "Developers",
# }
# ]
compare_result = {}
missing_matches = 0
for gitlab_group in gitlab_groups:
name = gitlab_group['path']
exists = False
for gitea_group in gitea_groups:
if name == gitea_group['name']:
exists = True
break
if exists:
_info(f'GITLAB: Group "{name}" exists on both GitLab and Gitea')
else:
_warn(f'GITLAB: Group "{name}" exists on GitLab only')
missing_matches += 1
compare_result[name] = 0 if exists else 1
for gitea_group in gitea_groups:
name = gitea_group['name']
exists = False
for gitlab_group in gitlab_groups:
if name == gitlab_group['path']:
exists = True
break
if not exists:
_warn(f'GITEA: Group "{name}" exists on Gitea only')
compare_result[name] = 2
return compare_result, missing_matches
def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
pass
def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
pass
def create_missing_groups(gitlab_groups: list, gitea_groups: list):
for gitlab_group in gitlab_groups:
name = gitlab_group['path']
exists = False
for gitea_group in gitea_groups:
if name == gitea_group['name']:
exists = True
break
if not exists:
_info(f'Creating missing group "{name}" on Gitea...')
try:
create_gitea_org(name)
except Exception as e:
_error(f'Failed to create Gitea group "{name}": {e}')
def run_migration():
_info('Migrating GitLab groups...')
gitlab_groups = get_gitlab_groups()
gitea_groups = get_gitea_groups()
_debug(f'Groups on GitLab: {len(gitlab_groups)}')
_trace(f'Groups on GitLab: {gitlab_groups}')
_debug(f'Groups on Gitea: {len(gitea_groups)}')
_trace(f'Groups on Gitea: {gitea_groups}')
group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
if missing_matches > 0:
_warn(f'{missing_matches} groups are missing on Gitea!')
if missing_matches > 0 and CREATE_MISSING_GROUPS:
_info('Creating missing groups on Gitea...')
create_missing_groups(gitlab_groups, gitea_groups)
_info('Migrating GitLab users...')
gitlab_users = get_gitlab_users()
gitea_users = get_gitea_users()
cmp_gitlab_gitea_users(gitlab_users, gitea_users)
_info('Migrating GitLab projects...')
gitlab_projects = get_gitlab_projects()
gitea_projects = get_gitea_projects()
cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
def main():
_info('Gitlab2Gitea v1.0 - by Zion Networks')
_info('------------------------------------')
_info('')
if sys.version_info < (3, 6):
_error('Python 3.6 or higher is required!')
return
else:
_debug(f'Python version: {sys.version_info.major}.{sys.version_info.minor}')
if not GITLAB_TOKEN:
_error('GitLab access token is missing!')
return
if not GITEA_TOKEN:
_error('Gitea access token is missing!')
return
if LOG_FILE and not APPEND_LOG:
with open(LOG_FILE, 'w') as log_file:
log_file.write('')
_debug(f'GitLab URL: {GITLAB_URL}')
_debug(f'Gitea URL: {GITEA_URL}')
_debug(f'Logging to: {LOG_FILE}')
if DRY_RUN:
_warn('Running in dry-run mode!')
_info('Testing endpoints...')
try:
check_gitlab()
check_gitea()
except Exception as e:
_error(f'An error occurred: {e}')
return
_info('Starting migration...')
try:
run_migration()
except Exception as e:
_error(f'An error occurred: {e}')
if __name__ == '__main__':
main()