Config generation¶
As a temporary solution for vcspull not being able to generate Configuration through scanning directories or fetching them via API (e.g. gitlab, github, etc), you can write scripts to generate configs in the mean time.
Collect repos from Gitlab¶
Contributed by Andreas Schleifer (a.schleifer@bigpoint.net)
Limitation on both, no pagination support in either, so only returns the first page of repos (as of Feb 26th this is 100).
#!/usr/bin/env bash if [ -z "${GITLAB_TOKEN}" ]; then echo 'Please provide the environment variable $GITLAB_TOKEN' exit 1 fi if [ $# -lt 2 ]; then echo "Usage: $0 <gitlab_host> <gitlab_namespace> [</path/to/target.config>]" exit 1 fi prefix="$(pwd)" gitlab_host="${1}" namespace="${2}" config_file="${3:-./vcspull.yaml}" current_namespace_path="" curl --silent --show-error --header "Authorization: Bearer ${GITLAB_TOKEN}" "https://${gitlab_host}/api/v4/groups/${namespace}/projects?include_subgroups=true&per_page=100" \ | jq -r '.[]|.namespace.full_path + " " + .path' \ | LC_ALL=C sort \ | while read namespace_path reponame; do if [ "${current_namespace_path}" != "${namespace_path}" ]; then current_namespace_path="${namespace_path}" echo "${prefix}/${current_namespace_path}:" fi # simplified config not working - https://github.com/vcs-python/vcspull/issues/332 #echo " ${reponame}: 'git+ssh://git@${gitlab_host}/${current_namespace_path}/${reponame}.git'" echo " ${reponame}:" echo " url: 'git+ssh://git@${gitlab_host}/${current_namespace_path}/${reponame}.git'" echo " remotes:" echo " origin: 'ssh://git@${gitlab_host}/${current_namespace_path}/${reponame}.git'" done \ | tee "${config_file}" $ env GITLAB_TOKEN=mySecretToken \ /path/to/generate_gitlab.sh gitlab.mycompany.com desired_namespace To be executed from the path where the repos should later be stored. It will use the current working directory as a “prefix” for the path used in the new config file.
Optional: Set config file output path as additional argument (will overwrite)
$ env GITLAB_TOKEN=mySecretToken \ /path/to/generate_gitlab.sh gitlab.mycompany.com desired_namespace /path/to/config.yaml Demonstration
Assume current directory of /home/user/workspace/ and script at /home/user/workspace/scripts/generate_gitlab.sh:
$ ./scripts/generate_gitlab.sh gitlab.com vcs-python New file vcspull.yaml:
/my/workspace/: g: url: "git+ssh://[email protected]/vcs-python/g.git" remotes: origin: "ssh://[email protected]/vcs-python/g.git" libvcs: url: "git+ssh://[email protected]/vcs-python/libvcs.git" remotes: origin: "ssh://[email protected]/vcs-python/libvcs.git" vcspull: url: "git+ssh://[email protected]/vcs-python/vcspull.git" remotes: origin: "ssh://[email protected]/vcs-python/vcspull.git" This confirms file overwrite, if already exists. It also requires passing the protocol/schema of the gitlab mirror, e.g. https://gitlab.com instead of gitlab.com.
#!/usr/bin/env python """Example script for export gitlab organization to vcspull config file.""" from __future__ import annotations import argparse import logging import os import pathlib import sys import typing as t import requests import yaml from libvcs.sync.git import GitRemote from vcspull.cli.sync import CouldNotGuessVCSFromURL, guess_vcs if t.TYPE_CHECKING: from vcspull.types import RawConfig log = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO, format="%(message)s") try: gitlab_token = os.environ["GITLAB_TOKEN"] except KeyError: log.info("Please provide the environment variable GITLAB_TOKEN") sys.exit(1) parser = argparse.ArgumentParser( description="Script to generate vcsconfig for all repositories \ under the given namespace (needs Gitlab >= 10.3)", ) parser.add_argument("gitlab_host", type=str, help="url to the gitlab instance") parser.add_argument( "gitlab_namespace", type=str, help="namespace/group in gitlab to generate vcsconfig for", ) parser.add_argument( "-c", type=str, help="path to the target config file (default: ./vcspull.yaml)", dest="config_file_name", required=False, default="./vcspull.yaml", ) args = vars(parser.parse_args()) gitlab_host = args["gitlab_host"] gitlab_namespace = args["gitlab_namespace"] config_filename = pathlib.Path(args["config_file_name"]) try: if config_filename.is_file(): result = input( f"The target config file ({config_filename}) already exists, \ do you want to overwrite it? [y/N] ", ) if result != "y": log.info( "Aborting per user request as existing config file (%s) should not be " "overwritten!", config_filename, ) sys.exit(0) config_file = config_filename.open(encoding="utf-8", mode="w") except OSError: log.info("File %s not accessible", config_filename) sys.exit(1) response = requests.get( f"{gitlab_host}/api/v4/groups/{gitlab_namespace}/projects", params={"include_subgroups": "true", "per_page": "100"}, headers={"Authorization": f"Bearer {gitlab_token}"}, ) if response.status_code != 200: log.info("Error: %s", response) sys.exit(1) path_prefix = pathlib.Path().cwd() config: RawConfig = {} for group in response.json(): url_to_repo = group["ssh_url_to_repo"].replace(":", "/") namespace_path = group["namespace"]["full_path"] reponame = group["path"] path = f"{path_prefix}/{namespace_path}" if path not in config: config[path] = {} # simplified config not working - https://github.com/vcs-python/vcspull/issues/332 # config[path][reponame] = 'git+ssh://%s' % (url_to_repo) vcs = guess_vcs(url_to_repo) if vcs is None: raise CouldNotGuessVCSFromURL(url_to_repo) config[path][reponame] = { "name": reponame, "path": path / reponame, "url": f"git+ssh://{url_to_repo}", "remotes": { "origin": GitRemote( name="origin", fetch_url=f"ssh://{url_to_repo}", push_url=f"ssh://{url_to_repo}", ), }, "vcs": vcs, } config_yaml = yaml.dump(config) log.info(config_yaml) config_file.write(config_yaml) config_file.close() Demonstration
Assume current directory of /home/user/workspace/ and script at /home/user/workspace/scripts/generate_gitlab.sh:
$ ./scripts/generate_gitlab.py https://gitlab.com vcs-python /my/workspace/vcs-python: g: remotes: origin: ssh://[email protected]/vcs-python/g.git url: git+ssh://[email protected]/vcs-python/g.git libvcs: remotes: origin: ssh://[email protected]/vcs-python/libvcs.git url: git+ssh://[email protected]/vcs-python/libvcs.git vcspull: remotes: origin: ssh://[email protected]/vcs-python/vcspull.git url: git+ssh://[email protected]/vcs-python/vcspull.git Contribute your own¶
Post yours on https://github.com/vcs-python/vcspull/discussions or create a PR to add yours to scripts/ and be featured here