Commit 3d941716 authored by Didier WECKMANN's avatar Didier WECKMANN
Browse files

feat(...): initial commit

feat(...): use pageing

chore(...): update url

feat(...): add sight clone mode

fix(...): typo

feat(...): better conan create support

feat(sight): cleanup directories before recreating default layout

fix(...): fix removing exisiting branches

fix(...): typo

fix(...): add git pull step

fix(...): add git pull step

fix(...): switch back to default branch if the current one has been deleted

feat(clone): allow https protocol


feat(...): remove deps


feat(sight): ressurect gitclone


chore(...): rename sight to gitclone

feat(gitclone): set executable flag

feat(gitclone): add tolerant mode, update sight mode


wip
parent 1ce68a3a
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import shlex
import shutil
import subprocess
import stat
import gitlab
def main():
"""
Main function
"""
# Query projects to clone
parsed_arguments = parse_arguments()
# Initialize gitlab API
gitlab_instance = initialize_gitlab(parsed_arguments)
print("command {0}".format(parsed_arguments.command))
if parsed_arguments.command == 'update':
update(parsed_arguments, gitlab_instance, parsed_arguments.path)
elif parsed_arguments.command == 'clone':
clone(parsed_arguments, gitlab_instance, parsed_arguments.path)
elif parsed_arguments.command == 'conan':
conan(parsed_arguments, gitlab_instance, parsed_arguments.path)
elif parsed_arguments.command == 'sight':
sight(parsed_arguments, gitlab_instance, parsed_arguments.path)
def parse_arguments(arguments=None):
"""
Parse command line arguments
Args:
arguments: optional argument list, to allow unit testing
Returns:
ArgumentParser object
"""
# Arguments parsing
parser = argparse.ArgumentParser(
description='Perform various git or conan task on Sight GitLab repositories.'
)
parser.add_argument(
'--gitlab_config',
type=str,
help='GitLab Python API configuration path. Default to ~/.python-gitlab.cfg'
)
parser.add_argument(
'--gitlab_config_section',
type=str,
default='ircad.fr',
help='GitLab Python API configuration section in configuration file.'
)
parser.add_argument(
'--gitlab_url',
type=str,
help='GitLab URL. Default to https://git.ircad.fr'
)
parser.add_argument(
'--gitlab_token',
type=str,
help='GitLab private access token.'
)
parser.add_argument(
'--https',
action='store_true',
default=False,
help='Use https for git shell command.'
)
parser.add_argument(
'--dry',
action='store_true',
default=False,
help='do nothing but print things.'
)
parser.add_argument(
'--force',
action='store_true',
default=False,
help='be careless and overwrite things like existing merge request.'
)
parser.add_argument(
'--verbose',
action='store_true',
default=False,
help='prints some stuff.'
)
parser.add_argument(
'--fast',
action='store_true',
default=False,
help='perform faster but less thorough operations'
)
parser.add_argument(
'--tolerant',
action='store_true',
default=False,
help='be somewhat tolerant with errors. Keep running if possible.'
)
parser.add_argument(
'--reset',
action='store_true',
default=False,
help='reset hard everything to pristine'
)
subparsers = parser.add_subparsers(
description='Command to execute',
dest='command'
)
update_subparser = subparsers.add_parser(
'update',
description='Update recursively git repositories or current directory if already inside a git repository'
)
clone_subparser = subparsers.add_parser(
'clone',
description='Clone recursively all GitLab projects'
)
clone_subparser.add_argument(
'--group',
type=str,
required=True,
help='group to look for projects.'
)
conan_subparser = subparsers.add_parser(
'conan',
description='Perform many conan operations'
)
conan_subparser.add_argument(
'--build',
action='store_true',
default=False,
help='build all conan packages.'
)
conan_subparser.add_argument(
'--clean',
action='store_true',
default=False,
help='clean all conan packages.'
)
sight_subparser = subparsers.add_parser(
'sight',
description='Perform many Sight operations'
)
sight_subparser.add_argument(
'--build',
action='store_true',
default=False,
help='build Sight.'
)
sight_subparser.add_argument(
'--clean',
action='store_true',
default=False,
help='clean Sight.'
)
def __readable_dir(directory):
if not os.path.isdir(directory):
raise argparse.ArgumentTypeError("readable_dir:{0} is a directory".format(directory))
if not os.access(directory, os.R_OK):
raise argparse.ArgumentTypeError("readable_dir:{0} is not readable".format(directory))
return directory
parser.add_argument(
'path',
type=__readable_dir,
default=os.getcwd(),
help='Path where to perform actions'
)
# arguments is not None if coming from tests
if arguments is None:
parsed_arguments = parser.parse_args()
else:
parsed_arguments = parser.parse_args(arguments)
return parsed_arguments
def initialize_gitlab(parsed_arguments):
"""
Return an initialized gitlab object
Args:
parsed_arguments: parsed arguments from command line
Returns:
GitLab instance object
"""
gitlab_config = parsed_arguments.gitlab_config
gitlab_url = parsed_arguments.gitlab_url
gitlab_token = parsed_arguments.gitlab_token
# Use default value for configuration if not given as argument
if not gitlab_config or not os.path.exists(gitlab_config):
gitlab_config = os.path.join(os.path.expanduser('~'), '.python-gitlab.cfg')
# Use default value for url if the url is not given as argument
if not gitlab_url:
gitlab_url = 'https://git.ircad.fr'
# Use default value for token if not given as argument
if not gitlab_token:
gitlab_token = os.getenv('GITLAB_TOKEN')
# If an url or a token has been specified or the config cannot be opened
if parsed_arguments.gitlab_url or parsed_arguments.gitlab_token or not os.path.exists(gitlab_config):
return gitlab.Gitlab(gitlab_url, private_token=gitlab_token, ssl_verify=False, api_version='4')
else:
return gitlab.Gitlab.from_config(parsed_arguments.gitlab_config_section, gitlab_config)
def update(parsed_arguments, gitlab_instance, root=os.getcwd()):
"""
Update all git repositories found at 'root'
Args:
parsed_arguments: parsed arguments from command line
gitlab_instance: the GitLab object instance
root: the root path where to begin the search
"""
if parsed_arguments.verbose:
print("Updating all git repository at '{0}'".format(root))
# Will contain all git repository
repositories = []
# Walk on all subdirectories
for path, directories, files in os.walk(root):
# Add the trailing '/' and use canonical path
path = os.path.join(os.path.realpath(path), '')
# Do not scan subdirectories of a git repository
def __is_already_scanned():
for _repository in repositories:
# Add the trailing '/' and use canonical path
_repository = os.path.join(os.path.realpath(_repository), '')
if os.path.commonprefix([path, _repository]) == _repository:
return True
return False
if __is_already_scanned():
continue
if check_git_repository(parsed_arguments, path):
if parsed_arguments.verbose:
print("Adding '{0}'".format(path))
repositories.append(path)
# Pull everything from remote
for repository in repositories:
update_repository(repository)
def update_repository(parsed_arguments, gitlab_instance, repository):
"""
Update a git reposity
Args:
parsed_arguments: parsed arguments from command line
gitlab_instance: the GitLab object instance
repository: the root path where to begin the search
"""
cloned = False
# Perform an hard reset if needed
if parsed_arguments.reset:
if parsed_arguments.dry or parsed_arguments.verbose:
print("cd {0} && git reset --hard && git clean -d -x --force ".format(repository))
if not parsed_arguments.dry:
result = subprocess.run(shlex.split('git reset --hard'), cwd=repository)
# Also remove everything not belonging to the repository
if result.returncode == 0:
result = subprocess.run(shlex.split('git clean -d -x --force'), cwd=repository)
if result.returncode != 0 and parsed_arguments.force:
# Reset failed, try to reclone everything
reclone(repository)
cloned = True
# Pulling everything from remote in Fast Forward only
if parsed_arguments.dry or parsed_arguments.verbose:
print("cd {0} && git pull --recurse-submodules --no-commit --ff-only --rebase --autostash --all".format(
repository))
if not parsed_arguments.dry:
result = subprocess.run(
shlex.split('git pull --recurse-submodules --no-commit --ff-only --rebase --autostash --all'),
cwd=repository)
# Fast Forward failed, try something else
if result.returncode != 0 and parsed_arguments.force:
if parsed_arguments.verbose:
print("cd {0} && git pull --recurse-submodules --all --force".format(repository))
result = subprocess.run(shlex.split('git pull --recurse-submodules --all --force'), cwd=repository)
# In last resort, ...
if result.returncode != 0:
# ...try to reclone everything
reclone(parsed_arguments, repository)
cloned = True
# Perform a final cleanup
if not cloned and not parsed_arguments.fast:
if parsed_arguments.dry or parsed_arguments.verbose:
print("cd {0} && git gc --aggressive --prune".format(repository))
if not parsed_arguments.dry:
subprocess.run(shlex.split('git gc --aggressive --prune'), cwd=repository, check=True)
def clone(parsed_arguments, gitlab_instance, root=os.getcwd()):
"""
Clone recursively all GitLab projects from a given group
Args:
parsed_arguments: parsed arguments from command line
gitlab_instance: the GitLab object instance
root: the root path where to begin the search
"""
if parsed_arguments.verbose:
print("Clone project(s) at '{0}'".format(root))
# Get the group
group = gitlab_instance.groups.get(parsed_arguments.group, lazy=True)
# Get the project list
group_projects = group.projects.list(as_list=False, lazy=True)
# For each project
for group_project in group_projects:
clone_project(parsed_arguments, gitlab_instance, group_project, root)
def clone_project(parsed_arguments, gitlab_instance, project, root=os.getcwd(), with_namespace=True):
"""
Clone or update a GitLab project
Args:
parsed_arguments: parsed arguments from command line
gitlab_instance: the GitLab object instance
project: the GitLab project to clone / update
root: the root path where to begin the search
with_namespace: use name or path_with_namespace for clone
"""
if with_namespace:
repository_path = os.path.join(root, project.path_with_namespace)
else:
repository_path = os.path.join(root, project.name)
if check_git_repository(parsed_arguments, repository_path):
print("Updating {0}...".format(project.name))
# Update repository
update_repository(parsed_arguments, gitlab_instance, repository_path)
else:
print("Cloning {0}...".format(project.name))
# Get the URL
if parsed_arguments.https:
origin_url = project.http_url_to_repo
else:
origin_url = project.ssh_url_to_repo
# Clone back
if parsed_arguments.dry or parsed_arguments.verbose:
print("git clone --shared --recurse-submodules {0} {1}".format(origin_url, repository_path))
if not parsed_arguments.dry:
subprocess.run(
shlex.split("git clone --shared --recurse-submodules {0} {1}".format(origin_url, repository_path)),
check=True)
def conan(parsed_arguments, gitlab_instance, root=os.getcwd()):
if parsed_arguments.verbose:
print("Perform Conan operations project(s) at '{0}'".format(root))
def sight(parsed_arguments, gitlab_instance, root=os.getcwd()):
"""
Clone/Update a sight environment
Args:
parsed_arguments: parsed arguments from command line
path: the path to the git repository
Returns:
True if 'path' point to an existing git repository, False if not
"""
if parsed_arguments.verbose:
print("Perform Sight operations project(s) at '{0}'".format(root))
paths = [
os.path.join(root, 'Sight/Src'),
os.path.join(root, 'Sight/Build/Release'),
os.path.join(root, 'Sight/Build/Debug'),
os.path.join(root, 'Sight/Install/Release'),
os.path.join(root, 'Sight/Install/Debug')
]
if parsed_arguments.reset:
if parsed_arguments.dry or parsed_arguments.verbose:
for path in paths:
print("shutil.rmtree('{0}')".format(path))
if not parsed_arguments.dry:
for path in paths:
shutil.rmtree(path, ignore_errors=True)
# Create sight directories
if parsed_arguments.dry or parsed_arguments.verbose:
for path in paths:
print("os.makedirs({0})".format(path))
if not parsed_arguments.dry:
for path in paths:
os.makedirs(path, exist_ok=True)
# Clone or update sight
clone_project(parsed_arguments, gitlab_instance, gitlab_instance.projects.get('Sight/sight'), paths[0], False)
# Clone or update sight-nonfree
clone_project(parsed_arguments, gitlab_instance, gitlab_instance.projects.get('Sight/non-free/sight-nonfree'),
paths[0], False)
# Create build scripts
release_build_script = os.path.join(paths[1], 'build.sh')
with open(release_build_script, 'w+') as script_file:
script_file.write('#!/bin/zsh\n')
script_file.write('setopt extended_glob\n')
script_file.write('rm -rf ^$(basename $0)\n')
script_file.write('unsetopt extended_glob\n')
script_file.write('cmake')
script_file.write(' ../../Src/sight')
script_file.write(' -G "CodeBlocks - Ninja"')
script_file.write(' -DCMAKE_BUILD_TYPE=Release')
script_file.write(' -DGENERATE_VSCODE_WS=ON')
script_file.write(' -DVERBOSE_CONAN=ON')
script_file.write(' -DCONAN_BUILD_MISSING=ON')
script_file.write(' && ninja\n\n')
# Set the script executable
os.chmod(release_build_script, os.stat(release_build_script).st_mode | stat.S_IEXEC)
debug_build_script = os.path.join(paths[2], 'build.sh')
with open(debug_build_script, 'w+') as script_file:
script_file.write('#!/bin/zsh\n')
script_file.write('setopt extended_glob\n')
script_file.write('rm -rf ^$(basename $0)\n')
script_file.write('unsetopt extended_glob\n')
script_file.write('cmake')
script_file.write(' ../../Src/sight')
script_file.write(' -G "CodeBlocks - Ninja"')
script_file.write(' -DCMAKE_BUILD_TYPE=Debug')
script_file.write(' -DGENERATE_VSCODE_WS=ON')
script_file.write(' -DVERBOSE_CONAN=ON')
script_file.write(' -DCONAN_BUILD_MISSING=ON')
script_file.write(' && ninja\n\n')
# Set the script executable
os.chmod(debug_build_script, os.stat(debug_build_script).st_mode | stat.S_IEXEC)
def check_git_repository(parsed_arguments, path):
"""
Test if we already have a repository for that name by fetching everything
Args:
parsed_arguments: parsed arguments from command line
path: the path to the git repository
Returns:
True if 'path' point to an existing git repository, False if not
"""
# Testing if the directory already exists, is a good start
if not os.path.isdir(path):
return False
# Test if there is a .git subdirectory
if not os.path.isdir(os.path.join(path, '.git')):
return False
if not parsed_arguments.fast:
# Test if we already have a repository with fsck
if parsed_arguments.dry or parsed_arguments.verbose:
print("git -C {0} fsck --full --strict".format(path))
if not parsed_arguments.dry:
if subprocess.run(shlex.split("git -C {0} fsck --full --strict".format(path))).returncode != 0:
return False
# Fetch all, as a second checking option and to be sure that remote are still there
if parsed_arguments.dry or parsed_arguments.verbose:
print("git -C {0} fetch --all --prune --recurse-submodules=yes".format(path))
if not parsed_arguments.dry:
if subprocess.run(shlex.split(
"git -C {0} fetch --all --prune --recurse-submodules=yes".format(path))).returncode != 0:
return False
return True
def reclone(parsed_arguments, repository):
"""
Clone again a damaged git repository
Args:
parsed_arguments: parsed arguments from command line
repository: old path to reclone
"""
# Capture the original URL
origin_url = subprocess.run(shlex.split('git config --get remote.origin.url'), cwd=repository, capture_output=True,
check=True).stdout.decode('UTF-8')
# Remove everything
if parsed_arguments.dry or parsed_arguments.verbose:
print("Delete {0}".format(repository))
if not parsed_arguments.dry:
shutil.rmtree(repository, ignore_errors=True)
# Clone back
if parsed_arguments.dry or parsed_arguments.verbose:
print("git clone --shared --recurse-submodules {0} {1}".format(origin_url, repository))
if not parsed_arguments.dry:
check = not parsed_arguments.tolerant
subprocess.run(shlex.split("git clone --shared --recurse-submodules {0} {1}".format(origin_url, repository)),
check=check)
if __name__ == "__main__":
main()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment