Further work on gitlab2gitea

gitlab2gitea/gitlab2gitea.py

@@ -15,8 +15,12 @@
 #   --gitea-url <GITEA_URL>                     Gitea URL
 #   --gitea-api-version <GITEA_API_VERSION>     Gitea API version (default: v1)
 #
+#   --create-missing-groups                     Create missing groups on Gitea (default: False)
+#   --create-missing-users                      Create missing users on Gitea (default: False)
+#
 #   --quiet                                     Enable quiet mode (default: False)
 #   --debug                                     Enable debug mode (default: False)
+#   --trace                                     Enable trace mode (default: False)
 #   --dry-run                                   Enable dry-run mode (default: False)
 #   --log-file <LOG_FILE>                       Log file (default: gitlab2gitea.log)
 #                                               If not set, logs will be only printed to stdout
@@ -41,9 +45,15 @@ GITEA_URL = ''
 GITEA_TOKEN = ''
 GITEA_API_VERSION = 'v1'
 
+# Settings - General Repository
+
+CREATE_MISSING_GROUPS = False
+CREATE_MISSING_USERS = False
+
 # Settings - General
 
 DEBUG = False
+TRACE = False
 DRY_RUN = False
 LOG_FILE = 'gitlab2gitea.log'
 APPEND_LOG = False
@@ -52,6 +62,7 @@ QUIET = False
 # Imports
 
 import os
+import sys
 import argparse
 import requests
 
@@ -78,12 +89,21 @@ if 'GITEA_TOKEN' in os.environ:
 if 'GITEA_API_VERSION' in os.environ:
     GITEA_API_VERSION = os.environ['GITEA_API_VERSION']
 
+if 'CREATE_MISSING_GROUPS' in os.environ:
+    CREATE_MISSING_GROUPS = bool(os.environ['CREATE_MISSING_GROUPS'])
+
+if 'CREATE_MISSING_USERS' in os.environ:
+    CREATE_MISSING_USERS = bool(os.environ['CREATE_MISSING_USERS'])
+
 if 'QUIET' in os.environ:
     QUIET = bool(os.environ['QUIET'])
 
 if 'DEBUG' in os.environ:
     DEBUG = bool(os.environ['DEBUG'])
 
+if 'TRACE' in os.environ:
+    TRACE = bool(os.environ['TRACE'])
+
 if 'DRY_RUN' in os.environ:
     DRY_RUN = bool(os.environ['DRY_RUN'])
 
@@ -122,14 +142,28 @@ if os.path.exists('.env'):
             if key == 'GITEA_API_VERSION':
                 GITEA_API_VERSION = value
             
+            if key == 'CREATE_MISSING_GROUPS':
+                if value.lower() == 'true' or value == '1':
+                    CREATE_MISSING_GROUPS = True
+            
+            if key == 'CREATE_MISSING_USERS':
+                if value.lower() == 'true' or value == '1':
+                    CREATE_MISSING_USERS = True
+
             if key == 'QUIET':
                 QUIET = bool(value)
 
             if key == 'DEBUG':
-                DEBUG = bool(value)
+                if value.lower() == 'true' or value == '1':
+                    DEBUG = True
+            
+            if key == 'TRACE':
+                if value.lower() == 'true' or value == '1':
+                    TRACE = True
 
             if key == 'DRY_RUN':
-                DRY_RUN = bool(value)
+                if value.lower() == 'true' or value == '1':
+                    DRY_RUN = True
 
             if key == 'LOG_FILE':
                 LOG_FILE = value
@@ -147,8 +181,11 @@ parser.add_argument('--gitea-token', help='Gitea access token')
 parser.add_argument('--gitlab-url', help='GitLab URL')
 parser.add_argument('--gitea-url', help='Gitea URL')
 parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
+parser.add_argument('--create-missing-groups', help='Create missing groups on Gitea', action='store_true')
+parser.add_argument('--create-missing-users', help='Create missing users on Gitea', action='store_true')
 parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
 parser.add_argument('--debug', help='Enable debug mode', action='store_true')
+parser.add_argument('--trace', help='Enable trace mode', action='store_true')
 parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true')
 parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log')
 parser.add_argument('--append-log', help='Append log file', action='store_true')
@@ -173,12 +210,21 @@ if args.gitea_url:
 if args.gitea_api_version:
     GITEA_API_VERSION = args.gitea_api_version
 
+if args.create_missing_groups:
+    CREATE_MISSING_GROUPS = True
+
+if args.create_missing_users:
+    CREATE_MISSING_USERS = True
+
 if args.quiet:
     QUIET = True
 
 if args.debug:
     DEBUG = True
 
+if args.trace:
+    TRACE = True
+
 if args.dry_run:
     DRY_RUN = True
 
@@ -195,8 +241,16 @@ GITEA_URL = GITEA_URL.rstrip('/')
 
 # Internal functions
 
+def _trace(message):
+    if TRACE:
+        print('\033[1m\033[36m[TRC]\033[0m', message)
+
+        if LOG_FILE:
+            with open(LOG_FILE, 'a') as log_file:
+                log_file.write(f'[TRC] {message}\n')
+
 def _debug(message):
-    if DEBUG:
+    if TRACE or DEBUG:
         print('\033[1m\033[34m[DBG]\033[0m', message)
 
         if LOG_FILE:
@@ -236,6 +290,8 @@ def check_gitlab():
         'Authorization': f'Bearer {GITLAB_TOKEN}'
     })
 
+    _trace(f'RESPONSE: {response.json()}')
+
     if response.status_code != 200:
         response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
         raise Exception(f'GitLab endpoint test failed: {response_message}')
@@ -252,6 +308,8 @@ def check_gitea():
         'Authorization': f'token {GITEA_TOKEN}'
     })
 
+    _trace(f'RESPONSE: {response.json()}')
+    
     if response.status_code != 200:
         response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
         raise Exception(f'Gitea endpoint test failed: {response_message}')
@@ -260,7 +318,30 @@ def check_gitea():
 
 # Endpoint: GET /api/{GITLAB_API_VERSION}/groups
 def get_gitlab_groups() -> list:
-    pass
+    groups = []
+
+    _debug(f'REQUEST: {GITLAB_URL}/api/{GITLAB_API_VERSION}/groups')
+
+    response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/groups',                    
+        params={
+            'all_available': 1,
+            'per_page': 100,
+            'page': 1,
+            'top_level_only': 1
+        },
+        headers={
+        'Content-Type': 'application/json',
+        'Accept': 'application/json',
+        'Authorization': f'Bearer {GITLAB_TOKEN}'
+    })
+
+    if response.status_code != 200:
+        response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
+        raise Exception(f'Failed to get GitLab groups: {response_message}')
+    else:
+        groups = response.json()
+
+    return groups
 
 # Endpoint: GET /api/{GITLAB_API_VERSION}/users
 def get_gitlab_users() -> list:
@@ -272,7 +353,24 @@ def get_gitlab_projects() -> list:
 
 # Endpoint: GET /api/{GITEA_API_VERSION}/orgs
 def get_gitea_groups() -> list:
-    pass
+    groups = []
+
+    _debug(f'REQUEST: {GITEA_URL}/api/{GITEA_API_VERSION}/orgs?all_available=1&per_page=100')
+
+    response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/orgs',
+        headers={
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'Authorization': f'token {GITEA_TOKEN}'
+        })
+
+    if response.status_code != 200:
+        response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
+        raise Exception(f'Failed to get Gitea groups: {response_message}')
+    else:
+        groups = response.json()
+
+    return groups
 
 # Endpoint: GET /api/{GITEA_API_VERSION}/users
 def get_gitea_users() -> list:
@@ -283,7 +381,110 @@ def get_gitea_projects() -> list:
     pass
 
 def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
-    pass
+
+    # 0 = exists on both
+    # 1 = exists on GitLab only
+    # 2 = exists on Gitea only
+    # planned structure:
+    # {
+    #   'groupname' : 0,
+    #   'groupname2' : 1,
+    #   'groupname3' : 2
+    # }
+
+    # Example for GitLab groups
+    # [
+    #     {
+    #         "id": 15,
+    #         "web_url": "https://git.madaboutpandas.com/groups/administration",
+    #         "name": "Administration",
+    #         "path": "administration",
+    #         "description": "",
+    #         "visibility": "private",
+    #         "share_with_group_lock": false,
+    #         "require_two_factor_authentication": false,
+    #         "two_factor_grace_period": 48,
+    #         "project_creation_level": "developer",
+    #         "auto_devops_enabled": null,
+    #         "subgroup_creation_level": "maintainer",
+    #         "emails_disabled": false,
+    #         "emails_enabled": true,
+    #         "mentions_disabled": null,
+    #         "lfs_enabled": true,
+    #         "math_rendering_limits_enabled": true,
+    #         "lock_math_rendering_limits_enabled": false,
+    #         "default_branch": null,
+    #         "default_branch_protection": 1,
+    #         "default_branch_protection_defaults": {
+    #             "allowed_to_push": [{"access_level": 30}],
+    #             "allow_force_push": false,
+    #             "allowed_to_merge": [{"access_level": 40}]
+    #         },
+    #         "avatar_url": null,
+    #         "request_access_enabled": true,
+    #         "full_name": "Administration",
+    #         "full_path": "administration",
+    #         "created_at": "2020-08-10T20:27:23.487Z",
+    #         "parent_id": null,
+    #         "organization_id": 1,
+    #         "shared_runners_setting": "enabled",
+    #         "ldap_cn": null,
+    #         "ldap_access": null,
+    #         "wiki_access_level": "enabled"
+    #     }
+    # ]
+
+    # Example for Gitea groups
+    # [
+    #     {
+    #         "id": 3,
+    #         "name": "YerbaBuena",
+    #         "full_name": "",
+    #         "email": "",
+    #         "avatar_url": "http://10.17.1.21/avatars/9f8ea65601abbf666adcec2b128180e4",
+    #         "description": "",
+    #         "website": "",
+    #         "location": "",
+    #         "visibility": "public",
+    #         "repo_admin_change_team_access": True,
+    #         "username": "YerbaBuena",
+    #     }
+    # ]
+
+    compare_result = {}
+    missing_matches = 0
+
+    for gitlab_group in gitlab_groups:
+        name = gitlab_group['path']
+        exists = False
+
+        for gitea_group in gitea_groups:
+            if name == gitea_group['name']:
+                exists = True
+                break
+
+        if exists:
+            _info(f'GITLAB: Group "{name}" exists on both GitLab and Gitea')
+        else:
+            _warn(f'GITLAB: Group "{name}" exists on GitLab only')
+            missing_matches += 1
+        
+        compare_result[name] = 0 if exists else 1
+    
+    for gitea_group in gitea_groups:
+        name = gitea_group['name']
+        exists = False
+
+        for gitlab_group in gitlab_groups:
+            if name == gitlab_group['path']:
+                exists = True
+                break
+
+        if not exists:
+            _warn(f'GITEA: Group "{name}" exists on Gitea only')
+            compare_result[name] = 2
+    
+    return compare_result, missing_matches
 
 def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
     pass
@@ -291,6 +492,9 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
 def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
     pass
 
+def create_missing_groups(gitlab_groups: list, gitea_groups: list):
+    pass
+
 def run_migration():
     
     _info('Migrating GitLab groups...')
@@ -298,7 +502,20 @@ def run_migration():
     gitlab_groups = get_gitlab_groups()
     gitea_groups = get_gitea_groups()
 
-    cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
+    _debug(f'Groups on GitLab: {len(gitlab_groups)}')
+    _trace(f'Groups on GitLab: {gitlab_groups}')
+    _debug(f'Groups on Gitea: {len(gitea_groups)}')
+    _trace(f'Groups on Gitea: {gitea_groups}')
+
+    group_result, missing_matches = cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
+
+    if missing_matches > 0:
+        _warn(f'{missing_matches} groups are missing on Gitea!')
+    
+    if missing_matches > 0 and CREATE_MISSING_GROUPS:
+        _info('Creating missing groups on Gitea...')
+
+        create_missing_groups(gitlab_groups, gitea_groups)
 
     _info('Migrating GitLab users...')
 
@@ -314,11 +531,16 @@ def run_migration():
 
     cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
 
-
-
 def main():
-    if DRY_RUN:
+    _info('Gitlab2Gitea v1.0 - by Zion Networks')
-        _warn('Running in dry-run mode!')
+    _info('------------------------------------')
+    _info('')
+
+    if sys.version_info < (3, 6):
+        _error('Python 3.6 or higher is required!')
+        return
+    else:
+        _debug(f'Python version: {sys.version_info.major}.{sys.version_info.minor}')
 
     if not GITLAB_TOKEN:
         _error('GitLab access token is missing!')
@@ -336,6 +558,9 @@ def main():
     _debug(f'Gitea URL: {GITEA_URL}')
     _debug(f'Logging to: {LOG_FILE}')
 
+    if DRY_RUN:
+        _warn('Running in dry-run mode!')
+
     _info('Testing endpoints...')
 
     try: