Finished project migrations, also added exit handler

gitlab2gitea/gitlab2gitea.py

@@ -25,9 +25,9 @@
 #   --no-update-existing-users                  Do not update existing users on Gitea (default: False)
 #   --no-update-existing-projects               Do not update existing projects on Gitea (default: False)
 #
-#   --include-wiki                              Include wiki repositories (default: False) - not implemented yet
+#   --include-wiki                              Include wiki repositories (default: False)
-#   --include-issues                            Include issues repositories (default: False) - not implemented yet
+#   --include-issues                            Include issues repositories (default: False)
-#   --include-merge-requests                    Include merge requests repositories (default: False) - not implemented yet
+#   --include-merge-requests                    Include merge requests repositories (default: False)
 #
 #   --override-groups                           Override existing groups on Gitea (default: False) - not implemented yet
 #   --override-users                            Override existing users on Gitea (default: False) - not implemented yet
@@ -98,11 +98,18 @@ LOG_FILE = "gitlab2gitea.log"
 APPEND_LOG = False
 QUIET = False
 
+### DEBUG - REMOVE LATER
+
+ONLY_ONE_PROJECT = False
+
+### END DEBUG
+
 # Internal variables - Do not change
 
 APP_NAME = "GitLab2Gitea"
 APP_VERSION = "1.0"
 APP_AUTHOR = "Zion Networks"
+EXIT_REQUESTED = False
 GITEA_RESERVED_USERNAMES = ["ghost", "notifications"]
 GITEA_RESERVED_ORGANAMES = [
     "api",
@@ -155,6 +162,7 @@ import argparse
 import requests
 import traceback
 import json
+import signal
 
 # Set cwd to script directory
 
@@ -776,6 +784,9 @@ def get_gitlab_projects() -> list:
     projects = []
 
     while next_page_link is not None:
+        if EXIT_REQUESTED:
+            return
+
         _debug(f'REQUEST: GET {next_page_link.split("?")[0]}')
         response = requests.get(
             next_page_link,
@@ -819,7 +830,7 @@ def migrate_gitlab_project_to_gitea(gitlab_project: dict):
     response = requests.post(
         f"{GITEA_URL}/api/{GITEA_API_VERSION}/repos/migrate",
         json={
-            "auth_token": GITEA_TOKEN,
+            "auth_token": GITLAB_TOKEN,
             "repo_name": gitlab_project["path"],
             "repo_owner": (
                 gitlab_project["namespace"]["path"]
@@ -828,7 +839,7 @@ def migrate_gitlab_project_to_gitea(gitlab_project: dict):
             ),
             "description": gitlab_project["description"],
             "clone_addr": gitlab_project["http_url_to_repo"],
-            "service": "gitlab",
+            "service": "git",
             "issues": INCLUDE_ISSUES,
             "pull_requests": INCLUDE_MERGE_REQUESTS,
             "wiki": INCLUDE_WIKI,
@@ -1193,17 +1204,26 @@ def get_gitea_user(username: str) -> dict:
 def remap_keys(d, mapping):
     def nested_get(d, keys):
         for key in keys.split("/"):
+            if EXIT_REQUESTED:
+                return
+
             d = d[key]
         return d
 
     def nested_set(d, keys, value):
         keys = keys.split("/")
         for key in keys[:-1]:
+            if EXIT_REQUESTED:
+                return
+
             d = d.setdefault(key, {})
         d[keys[-1]] = value
 
     remapped = {}
     for k, v in d.items():
+        if EXIT_REQUESTED:
+            return
+
         if k in mapping:
             nested_set(remapped, mapping[k], v)
         elif isinstance(v, dict):
@@ -1223,6 +1243,9 @@ def remap_keys(d, mapping):
 def is_ignored(path_to_check, ignore_dict):
     def nested_get(d, keys):
         for key in keys:
+            if EXIT_REQUESTED:
+                return
+
             if key in d:
                 d = d[key]
             else:
@@ -1250,6 +1273,9 @@ def cmp_dicts(
             all_keys = all_keys.intersection(to_check.keys())
 
         for key in all_keys:
+            if EXIT_REQUESTED:
+                return
+
             current_path = path + [key]
             if ignore and is_ignored(current_path, ignore):
                 continue
@@ -1303,10 +1329,16 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
     missing_matches = 0
 
     for gitlab_group in gitlab_groups:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_group["path"]
         exists = False
 
         for gitea_group in gitea_groups:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_group["name"]:
                 exists = True
                 break
@@ -1320,10 +1352,16 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
         compare_result[name] = 0 if exists else 1
 
     for gitea_group in gitea_groups:
+        if EXIT_REQUESTED:
+            return
+
         name = gitea_group["name"]
         exists = False
 
         for gitlab_group in gitlab_groups:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitlab_group["path"]:
                 exists = True
                 break
@@ -1341,10 +1379,16 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
     missing_matches = 0
 
     for gitlab_user in gitlab_users:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_user["username"]
         exists = False
 
         for gitea_user in gitea_users:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_user["login"]:
                 exists = True
                 break
@@ -1358,10 +1402,16 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
         compare_result[name] = 0 if exists else 1
 
     for gitea_user in gitea_users:
+        if EXIT_REQUESTED:
+            return
+
         name = gitea_user["login"]
         exists = False
 
         for gitlab_user in gitlab_users:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitlab_user["username"]:
                 exists = True
                 break
@@ -1379,11 +1429,17 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
     missing_matches = 0
 
     for gitlab_project in gitlab_projects:
+        if EXIT_REQUESTED:
+            return
+
         exists = False
         name = gitlab_project["path"]
         owner = gitlab_project["namespace"]["path"]
 
         for gitea_project in gitea_projects["data"]:
+            if EXIT_REQUESTED:
+                return
+
             # fmt: off
             if (name == gitea_project["name"] and owner == gitea_project["owner"]["username"]):
                 exists = True
@@ -1398,11 +1454,17 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
         compare_result[name] = 0 if exists else 1
 
     for gitea_project in gitea_projects["data"]:
+        if EXIT_REQUESTED:
+            return
+
         exists = False
         name = gitea_project["name"]
         owner = gitea_project["owner"]["username"]
 
         for gitlab_project in gitlab_projects:
+            if EXIT_REQUESTED:
+                return
+
             # fmt: off
             if name == gitlab_project["path"] and owner == gitlab_project["namespace"]["path"]:
                 exists = True
@@ -1419,10 +1481,16 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
 def create_missing_groups(gitlab_groups: list, gitea_groups: list):
 
     for gitlab_group in gitlab_groups:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_group["path"]
         exists = False
 
         for gitea_group in gitea_groups:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_group["name"]:
                 exists = True
                 break
@@ -1444,10 +1512,16 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list):
 def create_missing_users(gitlab_users: list, gitea_users: list):
 
     for gitlab_user in gitlab_users:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_user["username"]
         exists = False
 
         for gitea_user in gitea_users["data"]:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_user["login"]:
                 exists = True
                 break
@@ -1479,11 +1553,17 @@ def create_missing_users(gitlab_users: list, gitea_users: list):
 def create_missing_projects(gitlab_projects: list, gitea_projects: list):
 
     for gitlab_project in gitlab_projects:
+        if EXIT_REQUESTED:
+            return
+
         exists = False
         name = gitlab_project["path"]
         group = gitlab_project["namespace"]["path"]
 
         for gitea_project in gitea_projects["data"]:
+            if EXIT_REQUESTED:
+                return
+
             if (
                 name == gitea_project["name"]
                 and group == gitea_project["owner"]["username"]
@@ -1508,24 +1588,30 @@ def create_missing_projects(gitlab_projects: list, gitea_projects: list):
                 _info(f'Migrating Gitlab project "{name}" to Gitea...')
                 migrate_gitlab_project_to_gitea(gitlab_project)
 
-                if DEBUG:
+                if ONLY_ONE_PROJECT:
                     _warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
                     break
 
             except Exception as e:
                 _exception(f'Failed to create Gitea project "{name}": {e}', e)
 
-                if DEBUG:
+                if ONLY_ONE_PROJECT:
                     _warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
                     break
 
 
 def update_existing_groups(gitlab_groups: list, gitea_groups: list):
     for gitlab_group in gitlab_groups:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_group["path"]
         exists = False
 
         for gitea_group in gitea_groups:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_group["name"]:
                 exists = True
                 break
@@ -1555,10 +1641,16 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list):
 def update_existing_users(gitlab_users: list, gitea_users: list):
 
     for gitlab_user in gitlab_users:
+        if EXIT_REQUESTED:
+            return
+
         name = gitlab_user["username"]
         exists = False
 
         for gitea_user in gitea_users["data"]:
+            if EXIT_REQUESTED:
+                return
+
             if name == gitea_user["login"]:
                 exists = True
                 break
@@ -1591,11 +1683,17 @@ def update_existing_projects(gitlab_projects: list, gitea_projects: list):
 
     # update existing projects
     for gitlab_project in gitlab_projects:
+        if EXIT_REQUESTED:
+            return
+
         exists = False
         name = gitlab_project["path"]
         group = gitlab_project["namespace"]["path"]
 
         for gitea_project in gitea_projects["data"]:
+            if EXIT_REQUESTED:
+                return
+
             if (
                 name == gitea_project["name"]
                 and group == gitea_project["owner"]["username"]
@@ -1622,14 +1720,14 @@ def update_existing_projects(gitlab_projects: list, gitea_projects: list):
                 else:
                     update_gitea_project(gitlab_project)
 
-                if DEBUG:
+                if ONLY_ONE_PROJECT:
                     _warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
                     break
 
             except Exception as e:
                 _exception(f'Failed to update Gitea project "{name}": {e}', e)
 
-                if DEBUG:
+                if ONLY_ONE_PROJECT:
                     _warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
                     break
         else:
@@ -1693,6 +1791,10 @@ def migrate_groups():
         if not DRY_RUN:
             try:
                 create_missing_groups(gitlab_groups, gitea_groups)
+
+                if EXIT_REQUESTED:
+                    return
+
             except Exception as e:
                 _exception(f"Failed to create missing groups: {e}", e)
         else:
@@ -1706,6 +1808,10 @@ def migrate_groups():
         try:
             if not DRY_RUN:
                 update_existing_groups(gitlab_groups, gitea_groups)
+
+                if EXIT_REQUESTED:
+                    return
+
             else:
                 _warn(
                     "Dry-run mode enabled, skipping update of existing groups on Gitea..."
@@ -1739,6 +1845,10 @@ def migrate_users():
 
         if not DRY_RUN:
             create_missing_users(gitlab_users, gitea_users)
+
+            if EXIT_REQUESTED:
+                return
+
         else:
             _warn(
                 "Dry-run mode enabled, skipping creation of missing users on Gitea..."
@@ -1750,6 +1860,10 @@ def migrate_users():
         try:
             if not DRY_RUN:
                 update_existing_users(gitlab_users, gitea_users)
+
+                if EXIT_REQUESTED:
+                    return
+
             else:
                 _warn(
                     "Dry-run mode enabled, skipping update of existing users on Gitea..."
@@ -1764,15 +1878,28 @@ def migrate_projects():
         _warn("EXISTING PROJECTS WILL BE OVERRIDDEN!")
 
     gitlab_projects = get_gitlab_projects()
+
+    if EXIT_REQUESTED:
+        return
+
     gitea_projects = get_gitea_projects()
 
+    if EXIT_REQUESTED:
+        return
+
     # dump the projects to json files
     with open("gitlab_projects.json", "w") as f:
         json.dump(gitlab_projects, f, indent=4)
 
+    if EXIT_REQUESTED:
+        return
+
     with open("gitea_projects.json", "w") as f:
         json.dump(gitea_projects, f, indent=4)
 
+    if EXIT_REQUESTED:
+        return
+
     _info(f"Projects on GitLab: {len(gitlab_projects)}")
     _trace(f"Projects on GitLab: {gitlab_projects}")
     _info(f"Projects on Gitea: {len(gitea_projects)}")
@@ -1782,6 +1909,9 @@ def migrate_projects():
         project_result, missing_matches = cmp_gitlab_gitea_projects(
             gitlab_projects, gitea_projects
         )
+
+        if EXIT_REQUESTED:
+            return
     except Exception as e:
         _exception(f"Failed to compare GitLab and Gitea projects: {e}", e)
         return
@@ -1794,6 +1924,9 @@ def migrate_projects():
 
         if not DRY_RUN:
             create_missing_projects(gitlab_projects, gitea_projects)
+
+            if EXIT_REQUESTED:
+                return
         else:
             _warn(
                 "Dry-run mode enabled, skipping creation of missing projects on Gitea..."
@@ -1805,6 +1938,9 @@ def migrate_projects():
         try:
             if not DRY_RUN:
                 update_existing_projects(gitlab_projects, gitea_projects)
+
+                if EXIT_REQUESTED:
+                    return
             else:
                 _warn(
                     "Dry-run mode enabled, skipping update of existing projects on Gitea..."
@@ -1816,18 +1952,27 @@ def migrate_projects():
 def run_migration():
 
     if ONLY_GROUPS:
+        _warn("Skipping users!")
+        _warn("Skipping projects!")
+
         _info("Migrating GitLab groups...")
         migrate_groups()
         _info("Group migration completed!")
         return
 
     elif ONLY_USERS:
+        _warn("Skipping groups!")
+        _warn("Skipping projects!")
+
         _info("Migrating GitLab users...")
         migrate_users()
         _info("User migration completed!")
         return
 
     elif ONLY_PROJECTS:
+        _warn("Skipping groups!")
+        _warn("Skipping users!")
+
         _info("Migrating GitLab projects...")
         migrate_projects()
         _info("Project migration completed!")
@@ -1838,16 +1983,24 @@ def run_migration():
         migrate_groups()
         _info("Group migration completed!")
 
+        if EXIT_REQUESTED:
+            return
+
         _info("Migrating GitLab users...")
         migrate_users()
         _info("User migration completed!")
 
+        if EXIT_REQUESTED:
+            return
+
         _info("Migrating GitLab projects...")
         migrate_projects()
         _info("Project migration completed!")
 
 
 def main():
+    signal.signal(signal.SIGINT, signal_handler)
+
     header_info = f"{APP_NAME} v{APP_VERSION} - by {APP_AUTHOR}"
 
     _info(f"{header_info}")
@@ -1896,5 +2049,19 @@ def main():
         _exception(f"An error occurred: {e}", e)
 
 
+def signal_handler(sig, frame):
+    global EXIT_REQUESTED
+
+    if EXIT_REQUESTED:
+        _error("EXITING IMMEDIATELY!")
+        try:
+            sys.exit(1)
+        except SystemExit:
+            os._exit(1)
+    else:
+        _warn("EXIT REQUESTED! Please wait...")
+        EXIT_REQUESTED = True
+
+
 if __name__ == "__main__":
     main()