From 73b1e6d8eb0118b625f3bca67ee9a5238f873a4d Mon Sep 17 00:00:00 2001 From: Enrico Ludwig Date: Fri, 19 Jul 2024 10:19:31 +0200 Subject: [PATCH] WIP subgroups --- gitlab2gitea/gitlab2gitea.py | 163 ++++++++++++++++++++++++++++++----- 1 file changed, 142 insertions(+), 21 deletions(-) diff --git a/gitlab2gitea/gitlab2gitea.py b/gitlab2gitea/gitlab2gitea.py index 18015fd..411521e 100644 --- a/gitlab2gitea/gitlab2gitea.py +++ b/gitlab2gitea/gitlab2gitea.py @@ -33,6 +33,8 @@ # --override-users Override existing users on Gitea (default: False) - not implemented yet # --override-projects Override existing projects on Gitea (default: False) # +# --rewrite-subgroups Rewrite subgroups as organizations (default: False) - not implemented yet +# # --skip-empty-groups Skip empty groups (default: False) - not implemented yet # --skip-empty-projects Skip empty projects (default: False) - not implemented yet # @@ -85,6 +87,8 @@ OVERRIDE_EXISTING_GROUPS = False OVERRIDE_EXISTING_USERS = False OVERRIDE_EXISTING_PROJECTS = False +REWRITE_SUBGROUPS = False + ONLY_GROUPS = False ONLY_USERS = False ONLY_PROJECTS = False @@ -223,6 +227,9 @@ if "OVERWRITE_EXISTING_USERS" in os.environ: if "OVERWRITE_EXISTING_PROJECTS" in os.environ: OVERRIDE_EXISTING_PROJECTS = bool(os.environ["OVERRIDE_EXISTING_PROJECTS"]) +if "REWRITE_SUBGROUPS" in os.environ: + REWRITE_SUBGROUPS = bool(os.environ["REWRITE_SUBGROUPS"]) + if "ONLY_GROUPS" in os.environ: ONLY_GROUPS = bool(os.environ["ONLY_GROUPS"]) @@ -327,6 +334,10 @@ if os.path.exists(".env"): if value.lower() == "true" or value == "1": OVERRIDE_EXISTING_PROJECTS = True + if key == "REWRITE_SUBGROUPS": + if value.lower() == "true" or value == "1": + REWRITE_SUBGROUPS = True + if key == "ONLY_GROUPS": if value.lower() == "true" or value == "1": ONLY_GROUPS = True @@ -385,6 +396,7 @@ parser.add_argument("--include-merge-requests", help="Include merge requests rep parser.add_argument("--override-groups", help="Override existing groups on Gitea", action="store_true") parser.add_argument("--override-users", help="Override existing users on Gitea", action="store_true") parser.add_argument("--override-projects", help="Override existing projects on Gitea", action="store_true") +parser.add_argument("--rewrite-subgroups", help="Rewrite subgroups as organizations", action="store_true") parser.add_argument("--only-groups", help="Migrate only groups", action="store_true") parser.add_argument("--only-users", help="Migrate only users", action="store_true") parser.add_argument("--only-projects", help="Migrate only projects", action="store_true") @@ -452,6 +464,9 @@ if args.override_users: if args.override_projects: OVERRIDE_EXISTING_PROJECTS = True +if args.rewrite_subgroups: + REWRITE_SUBGROUPS = True + if args.only_groups: ONLY_GROUPS = True @@ -538,6 +553,7 @@ GITEA_URL = GITEA_URL.rstrip("/") def _trace(message): + if TRACE: print("\033[1m\033[36m[TRC]\033[0m", message) @@ -547,6 +563,7 @@ def _trace(message): def _debug(message): + if TRACE or DEBUG: print("\033[1m\033[34m[DBG]\033[0m", message) @@ -556,6 +573,7 @@ def _debug(message): def _info(message): + print("\033[1m\033[32m[INF]\033[0m", message) if LOG_FILE: @@ -564,6 +582,7 @@ def _info(message): def _warn(message): + print("\033[1m\033[33m[WRN]\033[0m", message) if LOG_FILE: @@ -572,6 +591,7 @@ def _warn(message): def _error(message): + print("\033[1m\033[31m[ERR]\033[0m", message) if LOG_FILE: @@ -580,6 +600,7 @@ def _error(message): def _exception(exception, custom_message=None): + exc_type, exc_obj, exc_tb = sys.exc_info() filename = exc_tb.tb_frame.f_code.co_filename lineno = exc_tb.tb_lineno @@ -605,18 +626,22 @@ def _exception(exception, custom_message=None): def is_gitea_reserved_username(username: str) -> bool: + return username.lower() in [name.lower() for name in GITEA_RESERVED_USERNAMES] def is_gitea_reserved_organame(organame: str) -> bool: + return organame.lower() in [name.lower() for name in GITEA_RESERVED_ORGANAMES] def is_gitea_reserved_reponame(reponame: str) -> bool: + return reponame.lower() in [name.lower() for name in GITEA_RESERVED_REPONAMES] def is_gitlab_project_in_subgroup(project: dict) -> bool: + return ( project["namespace"]["kind"] == "group" and project["namespace"]["parent_id"] is not None @@ -624,6 +649,7 @@ def is_gitlab_project_in_subgroup(project: dict) -> bool: def gitlab2gitea_visibility(visibility: str) -> str: + if visibility == "private": return "private" elif visibility == "internal": @@ -636,6 +662,7 @@ def gitlab2gitea_visibility(visibility: str) -> str: # Endpoint: GET /api/{GITLAB_API_VERSION}/version def check_gitlab(): + _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/version") response = requests.get( @@ -662,6 +689,7 @@ def check_gitlab(): # Endpoint: GET /api/{GITEA_API_VERSION}/version def check_gitea(): + _debug(f"REQUEST: GET {GITEA_URL}/api/{GITEA_API_VERSION}/version") response = requests.get( @@ -687,14 +715,33 @@ def check_gitea(): # Endpoint: GET /api/{GITLAB_API_VERSION}/groups -def get_gitlab_groups() -> list: +def get_gitlab_subgroups(groups: dict = None) -> list: + + gitlab_groups = groups if groups else get_gitlab_groups(True) + subgroups = [] + + for group in gitlab_groups: + if group["parent_id"]: + subgroups.append(group) + + return subgroups + + +# Endpoint: GET /api/{GITLAB_API_VERSION}/groups +def get_gitlab_groups(with_subgroups: bool = False) -> list: + groups = [] _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups") response = requests.get( f"{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups", - params={"all_available": 1, "per_page": 100, "page": 1, "top_level_only": 1}, + params={ + "all_available": 1, + "per_page": 100, + "page": 1, + "top_level_only": 0 if with_subgroups else 1, + }, headers={ "Content-Type": "application/json", "Accept": "application/json", @@ -717,6 +764,7 @@ def get_gitlab_groups() -> list: # Endpoint: GET /api/{GITLAB_API_VERSION}/groups/{group_id} def get_gitlab_group(group_id: int) -> dict: + _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups/{group_id}") response = requests.get( @@ -925,6 +973,7 @@ def migrate_gitlab_user_to_gitea(user: dict): # Endpoint: GET /api/{GITEA_API_VERSION}/orgs def get_gitea_groups() -> list: + groups = [] _debug( @@ -1202,7 +1251,9 @@ def get_gitea_user(username: str) -> dict: def remap_keys(d, mapping): + def nested_get(d, keys): + for key in keys.split("/"): if EXIT_REQUESTED: return @@ -1211,6 +1262,7 @@ def remap_keys(d, mapping): return d def nested_set(d, keys, value): + keys = keys.split("/") for key in keys[:-1]: if EXIT_REQUESTED: @@ -1241,7 +1293,9 @@ def remap_keys(d, mapping): def is_ignored(path_to_check, ignore_dict): + def nested_get(d, keys): + for key in keys: if EXIT_REQUESTED: return @@ -1262,6 +1316,7 @@ def cmp_dicts( mapping: dict = None, ignore: dict = None, ) -> bool: + result = {} if mapping: @@ -1323,7 +1378,9 @@ def convert_gitlab_user_to_gitea(user: dict, extra_data: dict = None) -> dict: return gitea_user -def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict: +def cmp_gitlab_gitea_groups( + gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = [] +) -> dict: compare_result = {} missing_matches = 0 @@ -1478,7 +1535,9 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di return compare_result, missing_matches -def create_missing_groups(gitlab_groups: list, gitea_groups: list): +def create_missing_groups( + gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = [] +): for gitlab_group in gitlab_groups: if EXIT_REQUESTED: @@ -1508,6 +1567,35 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list): except Exception as e: _exception(f'Failed to create Gitea group "{name}": {e}', e) + for gitlab_group in gitlab_subgroups: + + if EXIT_REQUESTED: + return + + name = gitlab_group["path"] + exists = False + + for gitea_group in gitea_groups: + if EXIT_REQUESTED: + return + + if name == gitea_group["name"]: + exists = True + break + + if is_gitea_reserved_organame(name): + _warn(f'Skipping group "{name}": Group name is reserved on Gitea!') + continue + + if not exists: + _info(f'Creating missing group "{name}" on Gitea...') + + try: + _info(f'Migrating Gitlab group "{name}" to Gitea...') + migrate_gitlab_group_to_gitea(gitlab_group) + except Exception as e: + _exception(f'Failed to create Gitea group "{name}": {e}', e) + def create_missing_users(gitlab_users: list, gitea_users: list): @@ -1600,7 +1688,10 @@ def create_missing_projects(gitlab_projects: list, gitea_projects: list): break -def update_existing_groups(gitlab_groups: list, gitea_groups: list): +def update_existing_groups( + gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = [] +): + for gitlab_group in gitlab_groups: if EXIT_REQUESTED: return @@ -1637,6 +1728,43 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list): except Exception as e: _exception(f'Failed to update Gitea group "{name}": {e}', e) + for gitlab_group in gitlab_subgroups: + + if EXIT_REQUESTED: + return + + name = gitlab_group["path"] + exists = False + + for gitea_group in gitea_groups: + if EXIT_REQUESTED: + return + + if name == gitea_group["name"]: + exists = True + break + + if is_gitea_reserved_organame(name): + _warn(f'Skipping group "{name}": Group name is reserved on Gitea!') + continue + + if exists: + try: + _info(f'Updating existing group "{name}" on Gitea...') + update_gitea_org( + { + "path": gitlab_group["path"], + "description": gitlab_group["description"], + "website": gitlab_group["web_url"], + "visibility": gitlab2gitea_visibility( + gitlab_group["visibility"] + ), + "full_name": gitlab_group["full_name"], + }, + ) + except Exception as e: + _exception(f'Failed to update Gitea group "{name}": {e}', e) + def update_existing_users(gitlab_users: list, gitea_users: list): @@ -1767,16 +1895,19 @@ def delete_gitea_project(project: dict): def migrate_groups(): gitlab_groups = get_gitlab_groups() + gitlab_subgroups = get_gitlab_subgroups() if REWRITE_SUBGROUPS else [] gitea_groups = get_gitea_groups() _info(f"Groups on GitLab: {len(gitlab_groups)}") _trace(f"Groups on GitLab: {gitlab_groups}") + _info(f"Subgroups on GitLab: {len(gitlab_subgroups)}") + _trace(f"Subgroups on GitLab: {gitlab_subgroups}") _info(f"Groups on Gitea: {len(gitea_groups)}") _trace(f"Groups on Gitea: {gitea_groups}") try: group_result, missing_matches = cmp_gitlab_gitea_groups( - gitlab_groups, gitea_groups + gitlab_groups, gitea_groups, gitlab_subgroups ) except Exception as e: _exception(f"Failed to compare GitLab and Gitea groups: {e}", e) @@ -1790,7 +1921,7 @@ def migrate_groups(): if not DRY_RUN: try: - create_missing_groups(gitlab_groups, gitea_groups) + create_missing_groups(gitlab_groups, gitea_groups, gitlab_subgroups) if EXIT_REQUESTED: return @@ -1807,7 +1938,7 @@ def migrate_groups(): try: if not DRY_RUN: - update_existing_groups(gitlab_groups, gitea_groups) + update_existing_groups(gitlab_groups, gitea_groups, gitlab_subgroups) if EXIT_REQUESTED: return @@ -1821,6 +1952,7 @@ def migrate_groups(): def migrate_users(): + gitlab_users = get_gitlab_users() gitea_users = get_gitea_users() @@ -1884,19 +2016,6 @@ def migrate_projects(): gitea_projects = get_gitea_projects() - if EXIT_REQUESTED: - return - - # dump the projects to json files - with open("gitlab_projects.json", "w") as f: - json.dump(gitlab_projects, f, indent=4) - - if EXIT_REQUESTED: - return - - with open("gitea_projects.json", "w") as f: - json.dump(gitea_projects, f, indent=4) - if EXIT_REQUESTED: return @@ -1999,6 +2118,7 @@ def run_migration(): def main(): + signal.signal(signal.SIGINT, signal_handler) header_info = f"{APP_NAME} v{APP_VERSION} - by {APP_AUTHOR}" @@ -2050,6 +2170,7 @@ def main(): def signal_handler(sig, frame): + global EXIT_REQUESTED if EXIT_REQUESTED: