add: --create-missing-projects; started working on user migration; blueprinted further migration functions

gitlab2gitea/gitlab2gitea.py

@@ -15,8 +15,9 @@
 #   --gitea-url <GITEA_URL>                     Gitea URL
 #   --gitea-api-version <GITEA_API_VERSION>     Gitea API version (default: v1)
 #
-#   --create-missing-groups                     Create missing groups on Gitea (default: False)
+#   --create-missing-groups                     Create missing groups on Gitea (default: True)
-#   --create-missing-users                      Create missing users on Gitea (default: False)
+#   --create-missing-users                      Create missing users on Gitea (default: True)
+#   --create-missing-projects                   Create missing projects on Gitea (default: True)
 #
 #   --quiet                                     Enable quiet mode (default: False)
 #   --debug                                     Enable debug mode (default: False)
@@ -47,8 +48,9 @@ GITEA_API_VERSION = 'v1'
 
 # Settings - General Repository
 
-CREATE_MISSING_GROUPS = False
+CREATE_MISSING_GROUPS = True
-CREATE_MISSING_USERS = False
+CREATE_MISSING_USERS = True
+CREATE_MISSING_PROJECTS = True
 
 # Settings - General
 
@@ -65,6 +67,7 @@ import os
 import sys
 import argparse
 import requests
+import traceback
 
 # Set cwd to script directory
 
@@ -95,6 +98,9 @@ if 'CREATE_MISSING_GROUPS' in os.environ:
 if 'CREATE_MISSING_USERS' in os.environ:
     CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS'])
 
+if 'CREATE_MISSING_PROJECTS' in os.environ:
+    CREATE_MISSING_PROJECTS = bool(os.environ['CREATE_MISSING_PROJECTS'])
+
 if 'QUIET' in os.environ:
     QUIET = bool(os.environ['QUIET'])
 
@@ -150,6 +156,10 @@ if os.path.exists('.env'):
                 if value.lower() == 'true' or value == '1':
                     CREATE_MISSING_USERS = True
 
+            if key == 'CREATE_MISSING_PROJECTS':
+                if value.lower() == 'true' or value == '1':
+                    CREATE_MISSING_PROJECTS = True
+
             if key == 'QUIET':
                 QUIET = bool(value)
 
@@ -183,6 +193,7 @@ parser.add_argument('--gitea-url', help='Gitea URL')
 parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
 parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true')
 parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true')
+parser.add_argument('--create-missing-projects', help='Create missing projects on Gitea', action='store_true')
 parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
 parser.add_argument('--debug', help='Enable debug mode', action='store_true')
 parser.add_argument('--trace', help='Enable trace mode', action='store_true')
@@ -216,6 +227,9 @@ if args.create_missing_groups:
 if args.create_missing_users:
     CREATE_MISSING_USERS = True
 
+if args.create_missing_projects:
+    CREATE_MISSING_PROJECTS = True
+
 if args.quiet:
     QUIET = True
 
@@ -278,6 +292,27 @@ def _error(message):
         with open(LOG_FILE, 'a') as log_file:
             log_file.write(f'[ERR] {message}\n')
 
+def _exception(exception, custom_message=None):
+    exc_type, exc_obj, exc_tb = sys.exc_info()
+    filename = exc_tb.tb_frame.f_code.co_filename
+    lineno = exc_tb.tb_lineno
+    formatted_traceback = traceback.format_exc()
+
+    # Prepare the exception message
+    exception_message = (f'{custom_message}\n' if custom_message else '') + (
+        f'\033[1m\033[31m[EXC]\033[0m {exception} '
+        f'(file: {filename}, line: {lineno})\n'
+        f'{formatted_traceback}\n'
+    )
+
+    # Print the exception message to the console
+    print(exception_message)
+    
+    # Write the exception message to the log file if defined
+    if LOG_FILE:
+        with open(LOG_FILE, 'a') as log_file:
+            log_file.write(f'[EXC] {exception_message}\n')
+
 # PROGRAM
 
 def gitlab2gitea_visibility(visibility: str) -> str:
@@ -668,10 +703,146 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
     return compare_result, missing_matches
 
 def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
-    pass
+
+    # 0 = exists on both
+    # 1 = exists on GitLab only
+    # 2 = exists on Gitea only
+    # planned structure:
+    # {
+    #   'username' : 0,
+    #   'username2' : 1,
+    #   'username3' : 2
+    # }
+
+    # Structure on Gitlab
+    # [
+    #   {
+    #     "id": 3,
+    #     "username": "user.name",
+    #     "name": "user.name",
+    #     "state": "active",
+    #     "locked": false,
+    #     "avatar_url": "https://git.example.com/uploads/-/system/user/avatar/3/avatar.png",
+    #     "web_url": "https://git.example.com/user.name",
+    #     "created_at": "2020-07-16T14:43:36.801Z",
+    #     "bio": "",
+    #     "location": "",
+    #     "public_email": "",
+    #     "skype": "",
+    #     "linkedin": "",
+    #     "twitter": "",
+    #     "discord": "",
+    #     "website_url": "",
+    #     "organization": "",
+    #     "job_title": "",
+    #     "pronouns": null,
+    #     "bot": false,
+    #     "work_information": null,
+    #     "local_time": "10:25 AM",
+    #     "last_sign_in_at": "2024-06-30T20:05:29.531Z",
+    #     "confirmed_at": "2020-07-16T14:43:36.692Z",
+    #     "last_activity_on": "2024-07-14",
+    #     "email": "user.name@example.com",
+    #     "theme_id": 11,
+    #     "color_scheme_id": 2,
+    #     "projects_limit": 100000,
+    #     "current_sign_in_at": "2024-07-13T09:14:00.499Z",
+    #     "identities": [
+    #       {
+    #         "provider": "ldapmain",
+    #         "extern_uid": "cn=user.name,ou=internal,ou=users,ou=org,dc=example,dc=com",
+    #         "saml_provider_id": null
+    #       }
+    #     ],
+    #     "can_create_group": true,
+    #     "can_create_project": true,
+    #     "two_factor_enabled": true,
+    #     "external": false,
+    #     "private_profile": false,
+    #     "commit_email": "user.name@example.com",
+    #     "shared_runners_minutes_limit": null,
+    #     "extra_shared_runners_minutes_limit": null,
+    #     "scim_identities": [],
+    #     "is_admin": true,
+    #     "note": "",
+    #     "namespace_id": 3,
+    #     "created_by": null,
+    #     "email_reset_offered_at": null,
+    #     "using_license_seat": false
+    #   }
+    # ]
+
+    # Structure on Gitea
+    # {
+    #     "data": [
+    #         {
+    #             "id": 1,
+    #             "login": "user.name",
+    #             "login_name": "",
+    #             "source_id": 0,
+    #             "full_name": "",
+    #             "email": "user.name@example.com",
+    #             "avatar_url": "http://git.example.com/avatars/a81823ace1c9fa7ab59a61ca6e2c34b0",
+    #             "html_url": "http://git.example.com/user.name",
+    #             "language": "en-US",
+    #             "is_admin": true,
+    #             "last_login": "1970-01-01T01:00:00+01:00",
+    #             "created": "2024-07-13T11:04:57+02:00",
+    #             "restricted": false,
+    #             "active": true,
+    #             "prohibit_login": false,
+    #             "location": "",
+    #             "website": "",
+    #             "description": "",
+    #             "visibility": "private",
+    #             "followers_count": 0,
+    #             "following_count": 0,
+    #             "starred_repos_count": 0,
+    #             "username": "user.name"
+    #         }
+    #     ],
+    #     "ok": true
+    # }
+
+    compare_result = {}
+    missing_matches = 0
+
+    for gitlab_user in gitlab_users:
+        name = gitlab_user['username']
+        exists = False
+
+        for gitea_user in gitea_users:
+            if name == gitea_user['login']:
+                exists = True
+                break
+
+        if exists:
+            _info(f'GITLAB: User "{name}" exists on both GitLab and Gitea')
+        else:
+            _warn(f'GITLAB: User "{name}" exists on GitLab only')
+            missing_matches += 1
+        
+        compare_result[name] = 0 if exists else 1
+
+    for gitea_user in gitea_users:
+        name = gitea_user['login']
+        exists = False
+
+        for gitlab_user in gitlab_users:
+            if name == gitlab_user['username']:
+                exists = True
+                break
+
+        if not exists:
+            _warn(f'GITEA: User "{name}" exists on Gitea only')
+            compare_result[name] = 2
+
+    return compare_result, missing_matches
+
 
 def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
-    pass
+    
+    return {}, 0
 
 def create_missing_groups(gitlab_groups: list, gitea_groups: list):
     
@@ -690,7 +861,13 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list):
             try:
                 migrate_gitlab_group_to_gitea(gitlab_group)
             except Exception as e:
-                _error(f'Failed to create Gitea group "{name}": {e}')
+                _exception(f'Failed to create Gitea group "{name}": {e}', e)
+
+def create_missing_users(gitlab_users: list, gitea_users: list):
+    pass
+
+def create_missing_projects(gitlab_projects: list, gitea_projects: list):
+    pass
 
 def update_existing_groups(gitlab_groups: list, gitea_groups: list):
     for gitlab_group in gitlab_groups:
@@ -713,7 +890,10 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list):
                     'full_name': gitlab_group['full_name']
                 })
             except Exception as e:
-                _error(f'Failed to update Gitea group "{name}": {e}')
+                _exception(f'Failed to update Gitea group "{name}": {e}', e)
+
+def update_existing_users(gitlab_users: list, gitea_users: list):
+    pass
 
 def migrate_groups():
     gitlab_groups = get_gitlab_groups()
@@ -724,7 +904,11 @@ def migrate_groups():
     _info(f'Groups on Gitea: {len(gitea_groups)}')
     _trace(f'Groups on Gitea: {gitea_groups}')
 
+    try:
         group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
+    except Exception as e:
+        _exception(f'Failed to compare GitLab and Gitea groups: {e}', e)
+        return
 
     if missing_matches > 0:
         _warn(f'{missing_matches} groups are missing on Gitea!')
@@ -733,7 +917,10 @@ def migrate_groups():
         _info('Creating missing groups on Gitea...')
 
         if not DRY_RUN:
+            try:
                 create_missing_groups(gitlab_groups, gitea_groups)
+            except Exception as e:
+                _exception(f'Failed to create missing groups: {e}', e)
         else:
             _warn('Dry-run mode enabled, skipping creation of missing groups on Gitea...')
 
@@ -745,7 +932,7 @@ def migrate_groups():
         else:
             _warn('Dry-run mode enabled, skipping update of existing groups on Gitea...')
     except Exception as e:
-        _error(f'Failed to update existing groups: {e}')
+        _exception(f'Failed to update existing groups: {e}', e)
 
 def migrate_users():
     gitlab_users = get_gitlab_users()
@@ -756,7 +943,33 @@ def migrate_users():
     _info(f'Users on Gitea: {len(gitea_users)}')
     _trace(f'Users on Gitea: {gitea_users}')
 
-    cmp_gitlab_gitea_users(gitlab_users, gitea_users)
+    try:
+        user_result, missing_matches = cmp_gitlab_gitea_users(gitlab_users, gitea_users['data'])
+    except Exception as e:
+        _exception(f'Failed to compare GitLab and Gitea users: {e}', e)
+        return
+    
+    if missing_matches > 0:
+        _warn(f'{missing_matches} users are missing on Gitea!')
+
+    if missing_matches > 0 and CREATE_MISSING_USERS:
+        _info('Creating missing users on Gitea...')
+
+        if not DRY_RUN:
+            create_missing_users(gitlab_users, gitea_users)
+        else:
+            _warn('Dry-run mode enabled, skipping creation of missing users on Gitea...')
+
+    _info('Updating existing users on Gitea...')
+
+    try:
+        if not DRY_RUN:
+            update_existing_users(gitlab_users, gitea_users)
+        else:
+            _warn('Dry-run mode enabled, skipping update of existing users on Gitea...')
+    except Exception as e:
+        _exception(f'Failed to update existing users: {e}', e)
+
 
 def migrate_projects():
     gitlab_projects = get_gitlab_projects()
@@ -767,7 +980,22 @@ def migrate_projects():
     _info(f'Projects on Gitea: {len(gitea_projects)}')
     _trace(f'Projects on Gitea: {gitea_projects}')
 
-    cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
+    try:
+        project_result, missing_matches = cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
+    except Exception as e:
+        _exception(f'Failed to compare GitLab and Gitea projects: {e}', e)
+        return
+    
+    if missing_matches > 0:
+        _warn(f'{missing_matches} projects are missing on Gitea!')
+
+    if missing_matches > 0 and CREATE_MISSING_PROJECTS:
+        _info('Creating missing projects on Gitea...')
+
+        if not DRY_RUN:
+            create_missing_projects(gitlab_projects, gitea_projects)
+        else:
+            _warn('Dry-run mode enabled, skipping creation of missing projects on Gitea...')
 
 def run_migration():
     
@@ -816,7 +1044,7 @@ def main():
         check_gitlab()
         check_gitea()
     except Exception as e:
-        _error(f'An error occurred: {e}')
+        _exception(f'An error occurred: {e}', e)
         return
     
     _info('Starting migration...')
@@ -824,7 +1052,7 @@ def main():
     try:
         run_migration()
     except Exception as e:
-        _error(f'An error occurred: {e}')
+        _exception(f'An error occurred: {e}', e)
 
 if __name__ == '__main__':
     main()