Finished project migrations, also added exit handler
This commit is contained in:
parent
66db60e2ef
commit
603f39bc6f
@ -25,9 +25,9 @@
|
||||
# --no-update-existing-users Do not update existing users on Gitea (default: False)
|
||||
# --no-update-existing-projects Do not update existing projects on Gitea (default: False)
|
||||
#
|
||||
# --include-wiki Include wiki repositories (default: False) - not implemented yet
|
||||
# --include-issues Include issues repositories (default: False) - not implemented yet
|
||||
# --include-merge-requests Include merge requests repositories (default: False) - not implemented yet
|
||||
# --include-wiki Include wiki repositories (default: False)
|
||||
# --include-issues Include issues repositories (default: False)
|
||||
# --include-merge-requests Include merge requests repositories (default: False)
|
||||
#
|
||||
# --override-groups Override existing groups on Gitea (default: False) - not implemented yet
|
||||
# --override-users Override existing users on Gitea (default: False) - not implemented yet
|
||||
@ -98,11 +98,18 @@ LOG_FILE = "gitlab2gitea.log"
|
||||
APPEND_LOG = False
|
||||
QUIET = False
|
||||
|
||||
### DEBUG - REMOVE LATER
|
||||
|
||||
ONLY_ONE_PROJECT = False
|
||||
|
||||
### END DEBUG
|
||||
|
||||
# Internal variables - Do not change
|
||||
|
||||
APP_NAME = "GitLab2Gitea"
|
||||
APP_VERSION = "1.0"
|
||||
APP_AUTHOR = "Zion Networks"
|
||||
EXIT_REQUESTED = False
|
||||
GITEA_RESERVED_USERNAMES = ["ghost", "notifications"]
|
||||
GITEA_RESERVED_ORGANAMES = [
|
||||
"api",
|
||||
@ -155,6 +162,7 @@ import argparse
|
||||
import requests
|
||||
import traceback
|
||||
import json
|
||||
import signal
|
||||
|
||||
# Set cwd to script directory
|
||||
|
||||
@ -776,6 +784,9 @@ def get_gitlab_projects() -> list:
|
||||
projects = []
|
||||
|
||||
while next_page_link is not None:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
_debug(f'REQUEST: GET {next_page_link.split("?")[0]}')
|
||||
response = requests.get(
|
||||
next_page_link,
|
||||
@ -819,7 +830,7 @@ def migrate_gitlab_project_to_gitea(gitlab_project: dict):
|
||||
response = requests.post(
|
||||
f"{GITEA_URL}/api/{GITEA_API_VERSION}/repos/migrate",
|
||||
json={
|
||||
"auth_token": GITEA_TOKEN,
|
||||
"auth_token": GITLAB_TOKEN,
|
||||
"repo_name": gitlab_project["path"],
|
||||
"repo_owner": (
|
||||
gitlab_project["namespace"]["path"]
|
||||
@ -828,7 +839,7 @@ def migrate_gitlab_project_to_gitea(gitlab_project: dict):
|
||||
),
|
||||
"description": gitlab_project["description"],
|
||||
"clone_addr": gitlab_project["http_url_to_repo"],
|
||||
"service": "gitlab",
|
||||
"service": "git",
|
||||
"issues": INCLUDE_ISSUES,
|
||||
"pull_requests": INCLUDE_MERGE_REQUESTS,
|
||||
"wiki": INCLUDE_WIKI,
|
||||
@ -1193,17 +1204,26 @@ def get_gitea_user(username: str) -> dict:
|
||||
def remap_keys(d, mapping):
|
||||
def nested_get(d, keys):
|
||||
for key in keys.split("/"):
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
d = d[key]
|
||||
return d
|
||||
|
||||
def nested_set(d, keys, value):
|
||||
keys = keys.split("/")
|
||||
for key in keys[:-1]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
d = d.setdefault(key, {})
|
||||
d[keys[-1]] = value
|
||||
|
||||
remapped = {}
|
||||
for k, v in d.items():
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if k in mapping:
|
||||
nested_set(remapped, mapping[k], v)
|
||||
elif isinstance(v, dict):
|
||||
@ -1223,6 +1243,9 @@ def remap_keys(d, mapping):
|
||||
def is_ignored(path_to_check, ignore_dict):
|
||||
def nested_get(d, keys):
|
||||
for key in keys:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if key in d:
|
||||
d = d[key]
|
||||
else:
|
||||
@ -1250,6 +1273,9 @@ def cmp_dicts(
|
||||
all_keys = all_keys.intersection(to_check.keys())
|
||||
|
||||
for key in all_keys:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
current_path = path + [key]
|
||||
if ignore and is_ignored(current_path, ignore):
|
||||
continue
|
||||
@ -1303,10 +1329,16 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
|
||||
missing_matches = 0
|
||||
|
||||
for gitlab_group in gitlab_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_group["path"]
|
||||
exists = False
|
||||
|
||||
for gitea_group in gitea_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_group["name"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1320,10 +1352,16 @@ def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
|
||||
compare_result[name] = 0 if exists else 1
|
||||
|
||||
for gitea_group in gitea_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitea_group["name"]
|
||||
exists = False
|
||||
|
||||
for gitlab_group in gitlab_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitlab_group["path"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1341,10 +1379,16 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
|
||||
missing_matches = 0
|
||||
|
||||
for gitlab_user in gitlab_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_user["username"]
|
||||
exists = False
|
||||
|
||||
for gitea_user in gitea_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_user["login"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1358,10 +1402,16 @@ def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
|
||||
compare_result[name] = 0 if exists else 1
|
||||
|
||||
for gitea_user in gitea_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitea_user["login"]
|
||||
exists = False
|
||||
|
||||
for gitlab_user in gitlab_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitlab_user["username"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1379,11 +1429,17 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
|
||||
missing_matches = 0
|
||||
|
||||
for gitlab_project in gitlab_projects:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
exists = False
|
||||
name = gitlab_project["path"]
|
||||
owner = gitlab_project["namespace"]["path"]
|
||||
|
||||
for gitea_project in gitea_projects["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
# fmt: off
|
||||
if (name == gitea_project["name"] and owner == gitea_project["owner"]["username"]):
|
||||
exists = True
|
||||
@ -1398,11 +1454,17 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
|
||||
compare_result[name] = 0 if exists else 1
|
||||
|
||||
for gitea_project in gitea_projects["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
exists = False
|
||||
name = gitea_project["name"]
|
||||
owner = gitea_project["owner"]["username"]
|
||||
|
||||
for gitlab_project in gitlab_projects:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
# fmt: off
|
||||
if name == gitlab_project["path"] and owner == gitlab_project["namespace"]["path"]:
|
||||
exists = True
|
||||
@ -1419,10 +1481,16 @@ def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> di
|
||||
def create_missing_groups(gitlab_groups: list, gitea_groups: list):
|
||||
|
||||
for gitlab_group in gitlab_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_group["path"]
|
||||
exists = False
|
||||
|
||||
for gitea_group in gitea_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_group["name"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1444,10 +1512,16 @@ def create_missing_groups(gitlab_groups: list, gitea_groups: list):
|
||||
def create_missing_users(gitlab_users: list, gitea_users: list):
|
||||
|
||||
for gitlab_user in gitlab_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_user["username"]
|
||||
exists = False
|
||||
|
||||
for gitea_user in gitea_users["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_user["login"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1479,11 +1553,17 @@ def create_missing_users(gitlab_users: list, gitea_users: list):
|
||||
def create_missing_projects(gitlab_projects: list, gitea_projects: list):
|
||||
|
||||
for gitlab_project in gitlab_projects:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
exists = False
|
||||
name = gitlab_project["path"]
|
||||
group = gitlab_project["namespace"]["path"]
|
||||
|
||||
for gitea_project in gitea_projects["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if (
|
||||
name == gitea_project["name"]
|
||||
and group == gitea_project["owner"]["username"]
|
||||
@ -1508,24 +1588,30 @@ def create_missing_projects(gitlab_projects: list, gitea_projects: list):
|
||||
_info(f'Migrating Gitlab project "{name}" to Gitea...')
|
||||
migrate_gitlab_project_to_gitea(gitlab_project)
|
||||
|
||||
if DEBUG:
|
||||
if ONLY_ONE_PROJECT:
|
||||
_warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
_exception(f'Failed to create Gitea project "{name}": {e}', e)
|
||||
|
||||
if DEBUG:
|
||||
if ONLY_ONE_PROJECT:
|
||||
_warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
|
||||
break
|
||||
|
||||
|
||||
def update_existing_groups(gitlab_groups: list, gitea_groups: list):
|
||||
for gitlab_group in gitlab_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_group["path"]
|
||||
exists = False
|
||||
|
||||
for gitea_group in gitea_groups:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_group["name"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1555,10 +1641,16 @@ def update_existing_groups(gitlab_groups: list, gitea_groups: list):
|
||||
def update_existing_users(gitlab_users: list, gitea_users: list):
|
||||
|
||||
for gitlab_user in gitlab_users:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
name = gitlab_user["username"]
|
||||
exists = False
|
||||
|
||||
for gitea_user in gitea_users["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if name == gitea_user["login"]:
|
||||
exists = True
|
||||
break
|
||||
@ -1591,11 +1683,17 @@ def update_existing_projects(gitlab_projects: list, gitea_projects: list):
|
||||
|
||||
# update existing projects
|
||||
for gitlab_project in gitlab_projects:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
exists = False
|
||||
name = gitlab_project["path"]
|
||||
group = gitlab_project["namespace"]["path"]
|
||||
|
||||
for gitea_project in gitea_projects["data"]:
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
if (
|
||||
name == gitea_project["name"]
|
||||
and group == gitea_project["owner"]["username"]
|
||||
@ -1622,14 +1720,14 @@ def update_existing_projects(gitlab_projects: list, gitea_projects: list):
|
||||
else:
|
||||
update_gitea_project(gitlab_project)
|
||||
|
||||
if DEBUG:
|
||||
if ONLY_ONE_PROJECT:
|
||||
_warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
_exception(f'Failed to update Gitea project "{name}": {e}', e)
|
||||
|
||||
if DEBUG:
|
||||
if ONLY_ONE_PROJECT:
|
||||
_warn("DEBUG MODE ENABLED - BREAKING AFTER FIRST PROJECT")
|
||||
break
|
||||
else:
|
||||
@ -1693,6 +1791,10 @@ def migrate_groups():
|
||||
if not DRY_RUN:
|
||||
try:
|
||||
create_missing_groups(gitlab_groups, gitea_groups)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
_exception(f"Failed to create missing groups: {e}", e)
|
||||
else:
|
||||
@ -1706,6 +1808,10 @@ def migrate_groups():
|
||||
try:
|
||||
if not DRY_RUN:
|
||||
update_existing_groups(gitlab_groups, gitea_groups)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
else:
|
||||
_warn(
|
||||
"Dry-run mode enabled, skipping update of existing groups on Gitea..."
|
||||
@ -1739,6 +1845,10 @@ def migrate_users():
|
||||
|
||||
if not DRY_RUN:
|
||||
create_missing_users(gitlab_users, gitea_users)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
else:
|
||||
_warn(
|
||||
"Dry-run mode enabled, skipping creation of missing users on Gitea..."
|
||||
@ -1750,6 +1860,10 @@ def migrate_users():
|
||||
try:
|
||||
if not DRY_RUN:
|
||||
update_existing_users(gitlab_users, gitea_users)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
else:
|
||||
_warn(
|
||||
"Dry-run mode enabled, skipping update of existing users on Gitea..."
|
||||
@ -1764,15 +1878,28 @@ def migrate_projects():
|
||||
_warn("EXISTING PROJECTS WILL BE OVERRIDDEN!")
|
||||
|
||||
gitlab_projects = get_gitlab_projects()
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
gitea_projects = get_gitea_projects()
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
# dump the projects to json files
|
||||
with open("gitlab_projects.json", "w") as f:
|
||||
json.dump(gitlab_projects, f, indent=4)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
with open("gitea_projects.json", "w") as f:
|
||||
json.dump(gitea_projects, f, indent=4)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
_info(f"Projects on GitLab: {len(gitlab_projects)}")
|
||||
_trace(f"Projects on GitLab: {gitlab_projects}")
|
||||
_info(f"Projects on Gitea: {len(gitea_projects)}")
|
||||
@ -1782,6 +1909,9 @@ def migrate_projects():
|
||||
project_result, missing_matches = cmp_gitlab_gitea_projects(
|
||||
gitlab_projects, gitea_projects
|
||||
)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
except Exception as e:
|
||||
_exception(f"Failed to compare GitLab and Gitea projects: {e}", e)
|
||||
return
|
||||
@ -1794,6 +1924,9 @@ def migrate_projects():
|
||||
|
||||
if not DRY_RUN:
|
||||
create_missing_projects(gitlab_projects, gitea_projects)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
else:
|
||||
_warn(
|
||||
"Dry-run mode enabled, skipping creation of missing projects on Gitea..."
|
||||
@ -1805,6 +1938,9 @@ def migrate_projects():
|
||||
try:
|
||||
if not DRY_RUN:
|
||||
update_existing_projects(gitlab_projects, gitea_projects)
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
else:
|
||||
_warn(
|
||||
"Dry-run mode enabled, skipping update of existing projects on Gitea..."
|
||||
@ -1816,18 +1952,27 @@ def migrate_projects():
|
||||
def run_migration():
|
||||
|
||||
if ONLY_GROUPS:
|
||||
_warn("Skipping users!")
|
||||
_warn("Skipping projects!")
|
||||
|
||||
_info("Migrating GitLab groups...")
|
||||
migrate_groups()
|
||||
_info("Group migration completed!")
|
||||
return
|
||||
|
||||
elif ONLY_USERS:
|
||||
_warn("Skipping groups!")
|
||||
_warn("Skipping projects!")
|
||||
|
||||
_info("Migrating GitLab users...")
|
||||
migrate_users()
|
||||
_info("User migration completed!")
|
||||
return
|
||||
|
||||
elif ONLY_PROJECTS:
|
||||
_warn("Skipping groups!")
|
||||
_warn("Skipping users!")
|
||||
|
||||
_info("Migrating GitLab projects...")
|
||||
migrate_projects()
|
||||
_info("Project migration completed!")
|
||||
@ -1838,16 +1983,24 @@ def run_migration():
|
||||
migrate_groups()
|
||||
_info("Group migration completed!")
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
_info("Migrating GitLab users...")
|
||||
migrate_users()
|
||||
_info("User migration completed!")
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
return
|
||||
|
||||
_info("Migrating GitLab projects...")
|
||||
migrate_projects()
|
||||
_info("Project migration completed!")
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
header_info = f"{APP_NAME} v{APP_VERSION} - by {APP_AUTHOR}"
|
||||
|
||||
_info(f"{header_info}")
|
||||
@ -1896,5 +2049,19 @@ def main():
|
||||
_exception(f"An error occurred: {e}", e)
|
||||
|
||||
|
||||
def signal_handler(sig, frame):
|
||||
global EXIT_REQUESTED
|
||||
|
||||
if EXIT_REQUESTED:
|
||||
_error("EXITING IMMEDIATELY!")
|
||||
try:
|
||||
sys.exit(1)
|
||||
except SystemExit:
|
||||
os._exit(1)
|
||||
else:
|
||||
_warn("EXIT REQUESTED! Please wait...")
|
||||
EXIT_REQUESTED = True
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user