WIP subgroups

gitlab2gitea/gitlab2gitea.py

@@ -33,6 +33,8 @@
 #   --override-users                            Override existing users on Gitea (default: False) - not implemented yet
 #   --override-projects                         Override existing projects on Gitea (default: False)
 #
+#   --rewrite-subgroups                         Rewrite subgroups as organizations (default: False) - not implemented yet
+#
 #   --skip-empty-groups                         Skip empty groups (default: False) - not implemented yet
 #   --skip-empty-projects                       Skip empty projects (default: False) - not implemented yet
 #
@@ -85,6 +87,8 @@ OVERRIDE_EXISTING_GROUPS = False
 OVERRIDE_EXISTING_USERS = False
 OVERRIDE_EXISTING_PROJECTS = False
 
+REWRITE_SUBGROUPS = False
+
 ONLY_GROUPS = False
 ONLY_USERS = False
 ONLY_PROJECTS = False
@@ -223,6 +227,9 @@ if "OVERWRITE_EXISTING_USERS" in os.environ:
 if "OVERWRITE_EXISTING_PROJECTS" in os.environ:
     OVERRIDE_EXISTING_PROJECTS = bool(os.environ["OVERRIDE_EXISTING_PROJECTS"])
 
+if "REWRITE_SUBGROUPS" in os.environ:
+    REWRITE_SUBGROUPS = bool(os.environ["REWRITE_SUBGROUPS"])
+
 if "ONLY_GROUPS" in os.environ:
     ONLY_GROUPS = bool(os.environ["ONLY_GROUPS"])
 
@@ -327,6 +334,10 @@ if os.path.exists(".env"):
                 if value.lower() == "true" or value == "1":
                     OVERRIDE_EXISTING_PROJECTS = True
 
+            if key == "REWRITE_SUBGROUPS":
+                if value.lower() == "true" or value == "1":
+                    REWRITE_SUBGROUPS = True
+
             if key == "ONLY_GROUPS":
                 if value.lower() == "true" or value == "1":
                     ONLY_GROUPS = True
@@ -385,6 +396,7 @@ parser.add_argument("--include-merge-requests", help="Include merge requests rep
 parser.add_argument("--override-groups", help="Override existing groups on Gitea", action="store_true")
 parser.add_argument("--override-users", help="Override existing users on Gitea", action="store_true")
 parser.add_argument("--override-projects", help="Override existing projects on Gitea", action="store_true")
+parser.add_argument("--rewrite-subgroups", help="Rewrite subgroups as organizations", action="store_true")
 parser.add_argument("--only-groups", help="Migrate only groups", action="store_true")
 parser.add_argument("--only-users", help="Migrate only users", action="store_true")
 parser.add_argument("--only-projects", help="Migrate only projects", action="store_true")
@@ -452,6 +464,9 @@ if args.override_users:
 if args.override_projects:
     OVERRIDE_EXISTING_PROJECTS = True
 
+if args.rewrite_subgroups:
+    REWRITE_SUBGROUPS = True
+
 if args.only_groups:
     ONLY_GROUPS = True
 
@@ -538,6 +553,7 @@ GITEA_URL = GITEA_URL.rstrip("/")
 
 
 def _trace(message):
+
     if TRACE:
         print("\033[1m\033[36m[TRC]\033[0m", message)
 
@@ -547,6 +563,7 @@ def _trace(message):
 
 
 def _debug(message):
+
     if TRACE or DEBUG:
         print("\033[1m\033[34m[DBG]\033[0m", message)
 
@@ -556,6 +573,7 @@ def _debug(message):
 
 
 def _info(message):
+
     print("\033[1m\033[32m[INF]\033[0m", message)
 
     if LOG_FILE:
@@ -564,6 +582,7 @@ def _info(message):
 
 
 def _warn(message):
+
     print("\033[1m\033[33m[WRN]\033[0m", message)
 
     if LOG_FILE:
@@ -572,6 +591,7 @@ def _warn(message):
 
 
 def _error(message):
+
     print("\033[1m\033[31m[ERR]\033[0m", message)
 
     if LOG_FILE:
@@ -580,6 +600,7 @@ def _error(message):
 
 
 def _exception(exception, custom_message=None):
+
     exc_type, exc_obj, exc_tb = sys.exc_info()
     filename = exc_tb.tb_frame.f_code.co_filename
     lineno = exc_tb.tb_lineno
@@ -605,18 +626,22 @@ def _exception(exception, custom_message=None):
 
 
 def is_gitea_reserved_username(username: str) -> bool:
+
     return username.lower() in [name.lower() for name in GITEA_RESERVED_USERNAMES]
 
 
 def is_gitea_reserved_organame(organame: str) -> bool:
+
     return organame.lower() in [name.lower() for name in GITEA_RESERVED_ORGANAMES]
 
 
 def is_gitea_reserved_reponame(reponame: str) -> bool:
+
     return reponame.lower() in [name.lower() for name in GITEA_RESERVED_REPONAMES]
 
 
 def is_gitlab_project_in_subgroup(project: dict) -> bool:
+
     return (
         project["namespace"]["kind"] == "group"
         and project["namespace"]["parent_id"] is not None
@@ -624,6 +649,7 @@ def is_gitlab_project_in_subgroup(project: dict) -> bool:
 
 
 def gitlab2gitea_visibility(visibility: str) -> str:
+
     if visibility == "private":
         return "private"
     elif visibility == "internal":
@@ -636,6 +662,7 @@ def gitlab2gitea_visibility(visibility: str) -> str:
 
 # Endpoint: GET /api/{GITLAB_API_VERSION}/version
 def check_gitlab():
+
     _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/version")
 
     response = requests.get(
@@ -662,6 +689,7 @@ def check_gitlab():
 
 # Endpoint: GET /api/{GITEA_API_VERSION}/version
 def check_gitea():
+
     _debug(f"REQUEST: GET {GITEA_URL}/api/{GITEA_API_VERSION}/version")
 
     response = requests.get(
@@ -687,14 +715,33 @@ def check_gitea():
 
 
 # Endpoint: GET /api/{GITLAB_API_VERSION}/groups
-def get_gitlab_groups() -> list:
+def get_gitlab_subgroups(groups: dict = None) -> list:
+
+    gitlab_groups = groups if groups else get_gitlab_groups(True)
+    subgroups = []
+
+    for group in gitlab_groups:
+        if group["parent_id"]:
+            subgroups.append(group)
+
+    return subgroups
+
+
+# Endpoint: GET /api/{GITLAB_API_VERSION}/groups
+def get_gitlab_groups(with_subgroups: bool = False) -> list:
+
     groups = []
 
     _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups")
 
     response = requests.get(
         f"{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups",
-        params={"all_available": 1, "per_page": 100, "page": 1, "top_level_only": 1},
+        params={
+            "all_available": 1,
+            "per_page": 100,
+            "page": 1,
+            "top_level_only": 0 if with_subgroups else 1,
+        },
         headers={
             "Content-Type": "application/json",
             "Accept": "application/json",
@@ -717,6 +764,7 @@ def get_gitlab_groups() -> list:
 
 # Endpoint: GET /api/{GITLAB_API_VERSION}/groups/{group_id}
 def get_gitlab_group(group_id: int) -> dict:
+
     _debug(f"REQUEST: GET {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups/{group_id}")
 
     response = requests.get(
@@ -925,6 +973,7 @@ def migrate_gitlab_user_to_gitea(user: dict):
 
 # Endpoint: GET /api/{GITEA_API_VERSION}/orgs
 def get_gitea_groups() -> list:
+
     groups = []
 
     _debug(
@@ -1202,7 +1251,9 @@ def get_gitea_user(username: str) -> dict:
 
 
 def remap_keys(d, mapping):
+
     def nested_get(d, keys):
+
         for key in keys.split("/"):
             if EXIT_REQUESTED:
                 return
@@ -1211,6 +1262,7 @@ def remap_keys(d, mapping):
         return d
 
     def nested_set(d, keys, value):
+
         keys = keys.split("/")
         for key in keys[:-1]:
             if EXIT_REQUESTED:
@@ -1241,7 +1293,9 @@ def remap_keys(d, mapping):
 
 
 def is_ignored(path_to_check, ignore_dict):
+
     def nested_get(d, keys):
+
         for key in keys:
             if EXIT_REQUESTED:
                 return
@@ -1262,6 +1316,7 @@ def cmp_dicts(
     mapping: dict = None,
     ignore: dict = None,
 ) -> bool:
+
     result = {}
 
     if mapping:
@@ -1323,7 +1378,9 @@ def convert_gitlab_user_to_gitea(user: dict, extra_data: dict = None) -> dict:
     return gitea_user
 
 
-def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
+def cmp_gitlab_gitea_groups(
+    gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = []
+) -> dict:
 
     compare_result = {}
     missing_matches = 0
@@ -1478,7 +1535,9 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
     return compare_result, missing_matches
 
 
-def create_missing_groups(gitlab_groups: list, gitea_groups: list):
+def create_missing_groups(
+    gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = []
+):
 
     for gitlab_group in gitlab_groups:
         if EXIT_REQUESTED:
@@ -1508,6 +1567,35 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list):
             except Exception as e:
                 _exception(f'Failed to create Gitea group "{name}": {e}', e)
 
+    for gitlab_group in gitlab_subgroups:
+
+        if EXIT_REQUESTED:
+            return
+
+        name = gitlab_group["path"]
+        exists = False
+
+        for gitea_group in gitea_groups:
+            if EXIT_REQUESTED:
+                return
+
+            if name == gitea_group["name"]:
+                exists = True
+                break
+
+        if is_gitea_reserved_organame(name):
+            _warn(f'Skipping group "{name}": Group name is reserved on Gitea!')
+            continue
+
+        if not exists:
+            _info(f'Creating missing group "{name}" on Gitea...')
+
+            try:
+                _info(f'Migrating Gitlab group "{name}" to Gitea...')
+                migrate_gitlab_group_to_gitea(gitlab_group)
+            except Exception as e:
+                _exception(f'Failed to create Gitea group "{name}": {e}', e)
+
 
 def create_missing_users(gitlab_users: list, gitea_users: list):
 
@@ -1600,7 +1688,10 @@ def create_missing_projects(gitlab_projects: list, gitea_projects: list):
                     break
 
 
-def update_existing_groups(gitlab_groups: list, gitea_groups: list):
+def update_existing_groups(
+    gitlab_groups: list, gitea_groups: list, gitlab_subgroups: list = []
+):
+
     for gitlab_group in gitlab_groups:
         if EXIT_REQUESTED:
             return
@@ -1637,6 +1728,43 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list):
             except Exception as e:
                 _exception(f'Failed to update Gitea group "{name}": {e}', e)
 
+    for gitlab_group in gitlab_subgroups:
+
+        if EXIT_REQUESTED:
+            return
+
+        name = gitlab_group["path"]
+        exists = False
+
+        for gitea_group in gitea_groups:
+            if EXIT_REQUESTED:
+                return
+
+            if name == gitea_group["name"]:
+                exists = True
+                break
+
+        if is_gitea_reserved_organame(name):
+            _warn(f'Skipping group "{name}": Group name is reserved on Gitea!')
+            continue
+
+        if exists:
+            try:
+                _info(f'Updating existing group "{name}" on Gitea...')
+                update_gitea_org(
+                    {
+                        "path": gitlab_group["path"],
+                        "description": gitlab_group["description"],
+                        "website": gitlab_group["web_url"],
+                        "visibility": gitlab2gitea_visibility(
+                            gitlab_group["visibility"]
+                        ),
+                        "full_name": gitlab_group["full_name"],
+                    },
+                )
+            except Exception as e:
+                _exception(f'Failed to update Gitea group "{name}": {e}', e)
+
 
 def update_existing_users(gitlab_users: list, gitea_users: list):
 
@@ -1767,16 +1895,19 @@ def delete_gitea_project(project: dict):
 def migrate_groups():
 
     gitlab_groups = get_gitlab_groups()
+    gitlab_subgroups = get_gitlab_subgroups() if REWRITE_SUBGROUPS else []
     gitea_groups = get_gitea_groups()
 
     _info(f"Groups on GitLab: {len(gitlab_groups)}")
     _trace(f"Groups on GitLab: {gitlab_groups}")
+    _info(f"Subgroups on GitLab: {len(gitlab_subgroups)}")
+    _trace(f"Subgroups on GitLab: {gitlab_subgroups}")
     _info(f"Groups on Gitea: {len(gitea_groups)}")
     _trace(f"Groups on Gitea: {gitea_groups}")
 
     try:
         group_result, missing_matches = cmp_gitlab_gitea_groups(
-            gitlab_groups, gitea_groups
+            gitlab_groups, gitea_groups, gitlab_subgroups
         )
     except Exception as e:
         _exception(f"Failed to compare GitLab and Gitea groups: {e}", e)
@@ -1790,7 +1921,7 @@ def migrate_groups():
 
         if not DRY_RUN:
             try:
-                create_missing_groups(gitlab_groups, gitea_groups)
+                create_missing_groups(gitlab_groups, gitea_groups, gitlab_subgroups)
 
                 if EXIT_REQUESTED:
                     return
@@ -1807,7 +1938,7 @@ def migrate_groups():
 
         try:
             if not DRY_RUN:
-                update_existing_groups(gitlab_groups, gitea_groups)
+                update_existing_groups(gitlab_groups, gitea_groups, gitlab_subgroups)
 
                 if EXIT_REQUESTED:
                     return
@@ -1821,6 +1952,7 @@ def migrate_groups():
 
 
 def migrate_users():
+
     gitlab_users = get_gitlab_users()
     gitea_users = get_gitea_users()
 
@@ -1884,19 +2016,6 @@ def migrate_projects():
 
     gitea_projects = get_gitea_projects()
 
-    if EXIT_REQUESTED:
-        return
-
-    # dump the projects to json files
-    with open("gitlab_projects.json", "w") as f:
-        json.dump(gitlab_projects, f, indent=4)
-
-    if EXIT_REQUESTED:
-        return
-
-    with open("gitea_projects.json", "w") as f:
-        json.dump(gitea_projects, f, indent=4)
-
     if EXIT_REQUESTED:
         return
 
@@ -1999,6 +2118,7 @@ def run_migration():
 
 
 def main():
+
     signal.signal(signal.SIGINT, signal_handler)
 
     header_info = f"{APP_NAME} v{APP_VERSION} - by {APP_AUTHOR}"
@@ -2050,6 +2170,7 @@ def main():
 
 
 def signal_handler(sig, frame):
+
     global EXIT_REQUESTED
 
     if EXIT_REQUESTED: