Moved scripts to separate folders, started working on GitLab to Gitea migration script
This commit is contained in:
parent
823548bab7
commit
ae0588e569
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
@ -0,0 +1 @@
|
||||
.env
|
||||
2
LICENSE
2
LICENSE
@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 ZionNetworks
|
||||
Copyright (c) 2024 Zion Networks UG
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
|
||||
10
README.md
10
README.md
@ -10,9 +10,9 @@ We're planning to release a growing amount of open source software, that is free
|
||||
|
||||
### What scripts can be found here?
|
||||
|
||||
| Name | Description | License | Current Version | Written in | Supported Distros | File |
|
||||
|------|-------------|---------|-----------------|------------|-------------------|------|
|
||||
| [Borgmatic Setup Tool](#borgmatic-setup-tool) | If you plan to use borg as backup solution, you should also take a look at [borgmatic](https://torsion.org/borgmatic/). It's a Python wrapper for the award winning backup tool [borgbackup](https://borgbackup.readthedocs.io/en/stable/index.html) that simplifies creating secure and reliable backups even more. You can even store your configurations in files. This script will do the setup for you to. | [MIT]([LICENSE](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/LICENSE)) | v1.2.1 | Bash | Debian and derivates | [bormatic_setup.sh](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh) |
|
||||
| Name | Description | License | Current Version | Written in | Supported Distros | File |
|
||||
| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | --------------- | ---------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| [Borgmatic Setup Tool](#borgmatic-setup-tool) | If you plan to use borg as backup solution, you should also take a look at [borgmatic](https://torsion.org/borgmatic/). It's a Python wrapper for the award winning backup tool [borgbackup](https://borgbackup.readthedocs.io/en/stable/index.html) that simplifies creating secure and reliable backups even more. You can even store your configurations in files. This script will do the setup for you to. | [MIT]([LICENSE](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/LICENSE)) | v1.2.1 | Bash | Debian and derivates | [bormatic_setup.sh](https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh) |
|
||||
|
||||
#### Borgmatic Setup Tool
|
||||
|
||||
@ -68,8 +68,8 @@ If you want the script to also upgrade the packages on your system, set the `--u
|
||||
###### Example: Backup local configurations and home directories
|
||||
|
||||
1. Get the script via `wget` or `curl`:
|
||||
1. `wget https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh`
|
||||
2. `curl -O https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic_setup.sh`
|
||||
1. `wget https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh`
|
||||
2. `curl -O https://git.zion-networks.de/ZionNetworks/linux-bash-scripts/src/branch/main/borgmatic/borgmatic_setup.sh`
|
||||
2. Set the execution flag using `chmod +x borgmatic_setup.sh`
|
||||
3. Run the script:
|
||||
|
||||
|
||||
356
gitlab2gitea/gitlab2gitea.py
Normal file
356
gitlab2gitea/gitlab2gitea.py
Normal file
@ -0,0 +1,356 @@
|
||||
# Description: Script to migrate repositories from GitLab to Gitea
|
||||
# Author: Enrico Ludwig <enrico.ludwig@zion-networks.de>
|
||||
# Version: 1.0
|
||||
# Date: 2021-01-06
|
||||
#
|
||||
# Precedence of settings: arguments > .env file > environment variables > default values
|
||||
#
|
||||
# Usage:
|
||||
# python3 gitlab2gitea.py [options]
|
||||
#
|
||||
# --gitlab-token <GITLAB_TOKEN> GitLab access token
|
||||
# --gitlab-url <GITLAB_URL> GitLab URL
|
||||
# --gitlab-api-version <GITLAB_API_VERSION> GitLab API version (default: v4)
|
||||
# --gitea-token <GITEA_TOKEN> Gitea access token
|
||||
# --gitea-url <GITEA_URL> Gitea URL
|
||||
# --gitea-api-version <GITEA_API_VERSION> Gitea API version (default: v1)
|
||||
#
|
||||
# --quiet Enable quiet mode (default: False)
|
||||
# --debug Enable debug mode (default: False)
|
||||
# --dry-run Enable dry-run mode (default: False)
|
||||
# --log-file <LOG_FILE> Log file (default: gitlab2gitea.log)
|
||||
# If not set, logs will be only printed to stdout
|
||||
# --append-log Append log file (default: False)
|
||||
#
|
||||
# Example:
|
||||
# python3 gitlab2gitea.py \
|
||||
# --gitlab-url <GITLAB_URL> \
|
||||
# --gitlab-token <GITLAB_TOKEN> \
|
||||
# --gitea-url <GITEA_URL> \
|
||||
# --gitea-token <GITEA_TOKEN>
|
||||
|
||||
# Settings - GitLab
|
||||
|
||||
GITLAB_URL = ''
|
||||
GITLAB_TOKEN = ''
|
||||
GITLAB_API_VERSION = 'v4'
|
||||
|
||||
# Settings - Gitea
|
||||
|
||||
GITEA_URL = ''
|
||||
GITEA_TOKEN = ''
|
||||
GITEA_API_VERSION = 'v1'
|
||||
|
||||
# Settings - General
|
||||
|
||||
DEBUG = False
|
||||
DRY_RUN = False
|
||||
LOG_FILE = 'gitlab2gitea.log'
|
||||
APPEND_LOG = False
|
||||
QUIET = False
|
||||
|
||||
# Imports
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import requests
|
||||
|
||||
# Set cwd to script directory
|
||||
|
||||
os.chdir(os.path.dirname(os.path.realpath(__file__)))
|
||||
|
||||
# Read environment variables
|
||||
if 'GITLAB_URL' in os.environ:
|
||||
GITLAB_URL = os.environ['GITLAB_URL']
|
||||
|
||||
if 'GITLAB_TOKEN' in os.environ:
|
||||
GITLAB_TOKEN = os.environ['GITLAB_TOKEN']
|
||||
|
||||
if 'GITLAB_API_VERSION' in os.environ:
|
||||
GITLAB_API_VERSION = os.environ['GITLAB_API_VERSION']
|
||||
|
||||
if 'GITEA_URL' in os.environ:
|
||||
GITEA_URL = os.environ['GITEA_URL']
|
||||
|
||||
if 'GITEA_TOKEN' in os.environ:
|
||||
GITEA_TOKEN = os.environ['GITEA_TOKEN']
|
||||
|
||||
if 'GITEA_API_VERSION' in os.environ:
|
||||
GITEA_API_VERSION = os.environ['GITEA_API_VERSION']
|
||||
|
||||
if 'QUIET' in os.environ:
|
||||
QUIET = bool(os.environ['QUIET'])
|
||||
|
||||
if 'DEBUG' in os.environ:
|
||||
DEBUG = bool(os.environ['DEBUG'])
|
||||
|
||||
if 'DRY_RUN' in os.environ:
|
||||
DRY_RUN = bool(os.environ['DRY_RUN'])
|
||||
|
||||
if 'LOG_FILE' in os.environ:
|
||||
LOG_FILE = os.environ['LOG_FILE']
|
||||
|
||||
if 'APPEND_LOG' in os.environ:
|
||||
APPEND_LOG = bool(os.environ['APPEND_LOG'])
|
||||
|
||||
# Read .env file if exists and override environment variables
|
||||
|
||||
if os.path.exists('.env'):
|
||||
with open('.env', 'r') as env_file:
|
||||
for line in env_file:
|
||||
# skip comments, empty lines and lines without '='
|
||||
if line.startswith('#') or not '=' in line:
|
||||
continue
|
||||
|
||||
key, value = line.strip().split('=')
|
||||
|
||||
if key == 'GITLAB_API_VERSION':
|
||||
GITLAB_API_VERSION = value
|
||||
|
||||
if key == 'GITLAB_URL':
|
||||
GITLAB_URL = value
|
||||
|
||||
if key == 'GITLAB_TOKEN':
|
||||
GITLAB_TOKEN = value
|
||||
|
||||
if key == 'GITEA_URL':
|
||||
GITEA_URL = value
|
||||
|
||||
if key == 'GITEA_TOKEN':
|
||||
GITEA_TOKEN = value
|
||||
|
||||
if key == 'GITEA_API_VERSION':
|
||||
GITEA_API_VERSION = value
|
||||
|
||||
if key == 'QUIET':
|
||||
QUIET = bool(value)
|
||||
|
||||
if key == 'DEBUG':
|
||||
DEBUG = bool(value)
|
||||
|
||||
if key == 'DRY_RUN':
|
||||
DRY_RUN = bool(value)
|
||||
|
||||
if key == 'LOG_FILE':
|
||||
LOG_FILE = value
|
||||
|
||||
if key == 'APPEND_LOG':
|
||||
APPEND_LOG = bool(value)
|
||||
|
||||
# Read arguments and override environment variables
|
||||
|
||||
parser = argparse.ArgumentParser(description='Script to migrate repositories from GitLab to Gitea')
|
||||
|
||||
parser.add_argument('--gitlab-api-version', help='GitLab API version', default='v4')
|
||||
parser.add_argument('--gitlab-token', help='GitLab access token')
|
||||
parser.add_argument('--gitea-token', help='Gitea access token')
|
||||
parser.add_argument('--gitlab-url', help='GitLab URL')
|
||||
parser.add_argument('--gitea-url', help='Gitea URL')
|
||||
parser.add_argument('--gitea-api-version', help='Gitea API version', default='v1')
|
||||
parser.add_argument('--quiet', help='Enable quiet mode', action='store_true')
|
||||
parser.add_argument('--debug', help='Enable debug mode', action='store_true')
|
||||
parser.add_argument('--dry-run', help='Enable dry-run mode', action='store_true')
|
||||
parser.add_argument('--log-file', help='Log file', default='gitlab2gitea.log')
|
||||
parser.add_argument('--append-log', help='Append log file', action='store_true')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.gitlab_api_version:
|
||||
GITLAB_API_VERSION = args.gitlab_api_version
|
||||
|
||||
if args.gitlab_token:
|
||||
GITLAB_TOKEN = args.gitlab_token
|
||||
|
||||
if args.gitlab_url:
|
||||
GITLAB_URL = args.gitlab_url
|
||||
|
||||
if args.gitea_token:
|
||||
GITEA_TOKEN = args.gitea_token
|
||||
|
||||
if args.gitea_url:
|
||||
GITEA_URL = args.gitea_url
|
||||
|
||||
if args.gitea_api_version:
|
||||
GITEA_API_VERSION = args.gitea_api_version
|
||||
|
||||
if args.quiet:
|
||||
QUIET = True
|
||||
|
||||
if args.debug:
|
||||
DEBUG = True
|
||||
|
||||
if args.dry_run:
|
||||
DRY_RUN = True
|
||||
|
||||
if args.log_file:
|
||||
LOG_FILE = args.log_file
|
||||
|
||||
if args.append_log:
|
||||
APPEND_LOG = True
|
||||
|
||||
# Remove trailing slashes from URLs
|
||||
|
||||
GITLAB_URL = GITLAB_URL.rstrip('/')
|
||||
GITEA_URL = GITEA_URL.rstrip('/')
|
||||
|
||||
# Internal functions
|
||||
|
||||
def _debug(message):
|
||||
if DEBUG:
|
||||
print('\033[1m\033[34m[DBG]\033[0m', message)
|
||||
|
||||
if LOG_FILE:
|
||||
with open(LOG_FILE, 'a') as log_file:
|
||||
log_file.write(f'[DBG] {message}\n')
|
||||
|
||||
def _info(message):
|
||||
print('\033[1m\033[32m[INF]\033[0m', message)
|
||||
|
||||
if LOG_FILE:
|
||||
with open(LOG_FILE, 'a') as log_file:
|
||||
log_file.write(f'[INF] {message}\n')
|
||||
|
||||
def _warn(message):
|
||||
print('\033[1m\033[33m[WRN]\033[0m', message)
|
||||
|
||||
if LOG_FILE:
|
||||
with open(LOG_FILE, 'a') as log_file:
|
||||
log_file.write(f'[WRN] {message}\n')
|
||||
|
||||
def _error(message):
|
||||
print('\033[1m\033[31m[ERR]\033[0m', message)
|
||||
|
||||
if LOG_FILE:
|
||||
with open(LOG_FILE, 'a') as log_file:
|
||||
log_file.write(f'[ERR] {message}\n')
|
||||
|
||||
# PROGRAM
|
||||
|
||||
# Endpoint: GET /api/{GITLAB_API_VERSION}/version
|
||||
def check_gitlab():
|
||||
_debug(f'REQUEST: {GITLAB_URL}/api/{GITLAB_API_VERSION}/version')
|
||||
|
||||
response = requests.get(f'{GITLAB_URL}/api/{GITLAB_API_VERSION}/version', headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
'Authorization': f'Bearer {GITLAB_TOKEN}'
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
|
||||
raise Exception(f'GitLab endpoint test failed: {response_message}')
|
||||
else:
|
||||
_info(f'GitLab endpoint version: {response.json()["version"]}')
|
||||
|
||||
# Endpoint: GET /api/{GITEA_API_VERSION}/version
|
||||
def check_gitea():
|
||||
_debug(f'REQUEST: {GITEA_URL}/api/{GITEA_API_VERSION}/version')
|
||||
|
||||
response = requests.get(f'{GITEA_URL}/api/{GITEA_API_VERSION}/version', headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
'Authorization': f'token {GITEA_TOKEN}'
|
||||
})
|
||||
|
||||
if response.status_code != 200:
|
||||
response_message = response.json()['message'] if 'message' in response.json() else 'Unknown error'
|
||||
raise Exception(f'Gitea endpoint test failed: {response_message}')
|
||||
else:
|
||||
_info(f'Gitea endpoint version: {response.json()["version"]}')
|
||||
|
||||
# Endpoint: GET /api/{GITLAB_API_VERSION}/groups
|
||||
def get_gitlab_groups() -> list:
|
||||
pass
|
||||
|
||||
# Endpoint: GET /api/{GITLAB_API_VERSION}/users
|
||||
def get_gitlab_users() -> list:
|
||||
pass
|
||||
|
||||
# Endpoint: GET /api/{GITLAB_API_VERSION}/projects
|
||||
def get_gitlab_projects() -> list:
|
||||
pass
|
||||
|
||||
# Endpoint: GET /api/{GITEA_API_VERSION}/orgs
|
||||
def get_gitea_groups() -> list:
|
||||
pass
|
||||
|
||||
# Endpoint: GET /api/{GITEA_API_VERSION}/users
|
||||
def get_gitea_users() -> list:
|
||||
pass
|
||||
|
||||
# Endpoint: GET /api/{GITEA_API_VERSION}/repos
|
||||
def get_gitea_projects() -> list:
|
||||
pass
|
||||
|
||||
def cmp_gitlab_gitea_groups(gitlab_groups: list, gitea_groups: list) -> dict:
|
||||
pass
|
||||
|
||||
def cmp_gitlab_gitea_users(gitlab_users: list, gitea_users: list) -> dict:
|
||||
pass
|
||||
|
||||
def cmp_gitlab_gitea_projects(gitlab_projects: list, gitea_projects: list) -> dict:
|
||||
pass
|
||||
|
||||
def run_migration():
|
||||
|
||||
_info('Migrating GitLab groups...')
|
||||
|
||||
gitlab_groups = get_gitlab_groups()
|
||||
gitea_groups = get_gitea_groups()
|
||||
|
||||
cmp_gitlab_gitea_groups(gitlab_groups, gitea_groups)
|
||||
|
||||
_info('Migrating GitLab users...')
|
||||
|
||||
gitlab_users = get_gitlab_users()
|
||||
gitea_users = get_gitea_users()
|
||||
|
||||
cmp_gitlab_gitea_users(gitlab_users, gitea_users)
|
||||
|
||||
_info('Migrating GitLab projects...')
|
||||
|
||||
gitlab_projects = get_gitlab_projects()
|
||||
gitea_projects = get_gitea_projects()
|
||||
|
||||
cmp_gitlab_gitea_projects(gitlab_projects, gitea_projects)
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
if DRY_RUN:
|
||||
_warn('Running in dry-run mode!')
|
||||
|
||||
if not GITLAB_TOKEN:
|
||||
_error('GitLab access token is missing!')
|
||||
return
|
||||
|
||||
if not GITEA_TOKEN:
|
||||
_error('Gitea access token is missing!')
|
||||
return
|
||||
|
||||
if LOG_FILE and not APPEND_LOG:
|
||||
with open(LOG_FILE, 'w') as log_file:
|
||||
log_file.write('')
|
||||
|
||||
_debug(f'GitLab URL: {GITLAB_URL}')
|
||||
_debug(f'Gitea URL: {GITEA_URL}')
|
||||
_debug(f'Logging to: {LOG_FILE}')
|
||||
|
||||
_info('Testing endpoints...')
|
||||
|
||||
try:
|
||||
check_gitlab()
|
||||
check_gitea()
|
||||
except Exception as e:
|
||||
_error(f'An error occurred: {e}')
|
||||
return
|
||||
|
||||
_info('Starting migration...')
|
||||
|
||||
try:
|
||||
run_migration()
|
||||
except Exception as e:
|
||||
_error(f'An error occurred: {e}')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user