From b4eacc471bf5b062c2e4af160de269b7825548fa Mon Sep 17 00:00:00 2001 From: Enrico Ludwig Date: Sun, 14 Jul 2024 11:28:23 +0200 Subject: [PATCH] Code cleanup / restructuring --- gitlab2gitea/gitlab2gitea.py | 132 +++++++++++++++++++++-------------- 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/gitlab2gitea/gitlab2gitea.py b/gitlab2gitea/gitlab2gitea.py index ebed5f4..1fd2c2f 100644 --- a/gitlab2gitea/gitlab2gitea.py +++ b/gitlab2gitea/gitlab2gitea.py @@ -375,51 +375,65 @@ def get_gitlab_group(group_id: int) -> dict: # Endpoint: GET /api/{GITLAB_API_VERSION}/users def get_gitlab_users() -> list: + next_page_link = f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/users' users = [] - _debug(f'REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/users') + while next_page_link is not None: + _debug(f'REQUEST: GET {next_page_link.split("?")[0]}') + response = requests.get(next_page_link, + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'Bearer {GITLAB_TOKEN}' + }) + + next_page_link = response.links['next']['url'] if 'next' in response.links else None - response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/users', - headers={ - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'Authorization': f'Bearer {GITLAB_TOKEN}' - }) - - _trace(f'RESPONSE: {response.json()}') + _trace(f'RESPONSE: {response.json()}') - if response.status_code != 200: - response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' - raise Exception(f'Failed to get GitLab users: {response_message}') - else: - users = response.json() + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'Failed to get GitLab users: {response_message}') + else: + users += response.json() return users # Endpoint: GET /api/{GITLAB_API_VERSION}/projects def get_gitlab_projects() -> list: + next_page_link = f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/projects' projects = [] - _debug(f'REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/projects') - - response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/projects', - headers={ - 'Content-Type': 'application/json', - 'Accept': 'application/json', - 'Authorization': f'Bearer {GITLAB_TOKEN}' - }) + while next_page_link is not None: + _debug(f'REQUEST: GET {next_page_link.split("?")[0]}') + response = requests.get(next_page_link, + params={ + 'all_available': 1, + 'per_page': 100 + }, + headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'Authorization': f'Bearer {GITLAB_TOKEN}' + }) + + next_page_link = response.links['next']['url'] if 'next' in response.links else None - _trace(f'RESPONSE: {response.json()}') + _trace(f'RESPONSE: {response.json()}') - if response.status_code != 200: - response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' - raise Exception(f'Failed to get GitLab projects: {response_message}') - else: - projects = response.json() + if response.status_code != 200: + response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error' + raise Exception(f'Failed to get GitLab projects: {response_message}') + else: + projects += response.json() return projects +# Endpoint: POST /api/{GITEA_API_VERSION}/admin/users +def migrate_gitlab_user_to_gitea(user: dict): + pass + # Endpoint: GET /api/{GITEA_API_VERSION}/orgs def get_gitea_groups() -> list: groups = [] @@ -490,19 +504,17 @@ def get_gitea_projects() -> list: return projects # Endpoint: POST /api/{GITEA_API_VERSION}/orgs -def create_gitea_org(name: str) -> dict: +def migrate_gitlab_group_to_gitea(gitlab_group: dict): - gitlab_group = get_gitlab_group(name) - if not gitlab_group: - raise Exception(f'GitLab group "{name}" does not exist!') + raise Exception('GitLab group is missing!') # Create Gitea group _debug(f'REQUEST: POST {GITEA_URL}/api/{GITEA_API_VERSION}/orgs') response = requests.post(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs', json={ - 'username': name, + 'username': gitlab_group['path'], 'full_name': gitlab_group['full_name'], 'description': gitlab_group['description'], 'website': gitlab_group['web_url'], @@ -524,9 +536,12 @@ def create_gitea_org(name: str) -> dict: return group # Endpoint: PATCH /api/{GITEA_API_VERSION}/orgs/{org} -def update_gitea_org(name: str, data: dict) -> dict: +def update_gitea_org(data: dict) -> dict: - # Update Gitea group + if not data: + raise Exception('Data is missing!') + + name = data['path'] _debug(f'REQUEST: PATCH {GITEA_URL}/api/{GITEA_API_VERSION}/orgs/{name}') @@ -673,7 +688,7 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list): _info(f'Creating missing group "{name}" on Gitea...') try: - create_gitea_org(name) + migrate_gitlab_group_to_gitea(gitlab_group) except Exception as e: _error(f'Failed to create Gitea group "{name}": {e}') @@ -700,16 +715,13 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list): except Exception as e: _error(f'Failed to update Gitea group "{name}": {e}') -def run_migration(): - - _info('Migrating GitLab groups...') - +def migrate_groups(): gitlab_groups = get_gitlab_groups() gitea_groups = get_gitea_groups() - _debug(f'Groups on GitLab: {len(gitlab_groups)}') + _info(f'Groups on GitLab: {len(gitlab_groups)}') _trace(f'Groups on GitLab: {gitlab_groups}') - _debug(f'Groups on Gitea: {len(gitea_groups)}') + _info(f'Groups on Gitea: {len(gitea_groups)}') _trace(f'Groups on Gitea: {gitea_groups}') group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups) @@ -719,39 +731,55 @@ def run_migration(): if missing_matches > 0 and CREATE_MISSING_GROUPS: _info('Creating missing groups on Gitea...') - create_missing_groups(gitlab_groups, gitea_groups) + + if not DRY_RUN: + create_missing_groups(gitlab_groups, gitea_groups) + else: + _warn('Dry-run mode enabled, skipping creation of missing groups on Gitea...') _info('Updating existing groups on Gitea...') try: - update_existing_groups(gitlab_groups, gitea_groups) + if not DRY_RUN: + update_existing_groups(gitlab_groups, gitea_groups) + else: + _warn('Dry-run mode enabled, skipping update of existing groups on Gitea...') except Exception as e: _error(f'Failed to update existing groups: {e}') - _info('Migrating GitLab users...') - +def migrate_users(): gitlab_users = get_gitlab_users() gitea_users = get_gitea_users() - _debug(f'Users on GitLab: {len(gitlab_users)}') + _info(f'Users on GitLab: {len(gitlab_users)}') _trace(f'Users on GitLab: {gitlab_users}') - _debug(f'Users on Gitea: {len(gitea_users)}') + _info(f'Users on Gitea: {len(gitea_users)}') _trace(f'Users on Gitea: {gitea_users}') cmp_gitlab_gitea_users(gitlab_users, gitea_users) - _info('Migrating GitLab projects...') - +def migrate_projects(): gitlab_projects = get_gitlab_projects() gitea_projects = get_gitea_projects() - _debug(f'Projects on GitLab: {len(gitlab_projects)}') + _info(f'Projects on GitLab: {len(gitlab_projects)}') _trace(f'Projects on GitLab: {gitlab_projects}') - _debug(f'Projects on Gitea: {len(gitea_projects)}') + _info(f'Projects on Gitea: {len(gitea_projects)}') _trace(f'Projects on Gitea: {gitea_projects}') cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects) +def run_migration(): + + _info('Migrating GitLab groups...') + migrate_groups() + + _info('Migrating GitLab users...') + migrate_users() + + _info('Migrating GitLab projects...') + migrate_projects() + def main(): _info('Gitlab2Gitea v1.0 - by Zion Networks') _info('------------------------------------')