From 60898368dbe7db2e60ee47b73238d9eb8bd67e20 Mon Sep 17 00:00:00 2001 From: Sindre Stephansen Date: Thu, 31 Mar 2022 20:42:56 +0200 Subject: [PATCH] Restructure the project so the repo and sync are run together Changes: - There is one unified docker-compose.yml for both repo and sync - Both containers are started together, but the maven container spins until the repo is ready - The repo is terminated when maven terminates - The repo is set up at port 80 in the container, 9001 on the host - The maven container is separated from the host network, only has connections to the repo --- README.md | 18 +--- docker-compose.yml | 21 ++++ repo/deploy.sh | 3 - repo/docker-compose.yml | 14 --- run.sh | 28 +++++ sync/.dockerignore | 1 + sync/Dockerfile | 5 +- sync/build-docker-image.sh | 5 - sync/resolve-deps-from-poms.sh | 12 ++- sync/settings.xml | 4 +- sync/sync.sh | 14 --- sync/wait-for-it.LICENCE | 20 ++++ sync/wait-for-it.sh | 182 +++++++++++++++++++++++++++++++++ 13 files changed, 268 insertions(+), 59 deletions(-) create mode 100644 docker-compose.yml delete mode 100755 repo/deploy.sh delete mode 100644 repo/docker-compose.yml create mode 100755 run.sh delete mode 100755 sync/build-docker-image.sh delete mode 100755 sync/sync.sh create mode 100644 sync/wait-for-it.LICENCE create mode 100755 sync/wait-for-it.sh diff --git a/README.md b/README.md index 24b496b..ba4af20 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,9 @@ # Maven Sync Setup -## Repository - _Reposilite_-based Maven Repository Manager acting as a caching proxy. ```sh -mkdir data -cd repo -./deploy.sh +./run.sh ``` -TODO: something is not working correctly when loading the initial configuration. -Take the content of the `reposilite.cdn` and paste it into the Configuration window -after logging in as `admin:secret` on http://localhost:9001 and press _Update configuration and reload_. - -## Sync job - -Script that can be triggered manually or scheduled. - -cd sync -./sync.sh -Mirror tarballs. +This starts the reposilite server, then starts maven and makes download all packages defined in the pom's in the `examples/` folder, including their dependencies. These packages are then cached by reposilite in `data/`. The relevant folders are turned into tarballs and saved in the root directory. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a41e737 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.9' + +services: + repo: + image: dzikoysk/reposilite:3.0.0-alpha.23 + user: ${USER_ID}:${GROUP_ID} + environment: + REPOSILITE_OPTS: "--token admin:secret --shared-configuration=/reposilite.cdn --shared-configuration-mode=copy" + volumes: + - ./repo/reposilite.cdn:/reposilite.cdn + - ./data:/app/data + ports: + - "9001:80" + restart: unless-stopped + maven-sync-job: + build: ./sync + depends_on: + - repo + volumes: + - ./examples:/poms + network_mode: "service:repo" diff --git a/repo/deploy.sh b/repo/deploy.sh deleted file mode 100755 index 80df2d0..0000000 --- a/repo/deploy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker-compose run -u $(id -u):$(id -g) --service-ports repo diff --git a/repo/docker-compose.yml b/repo/docker-compose.yml deleted file mode 100644 index 17cfbe1..0000000 --- a/repo/docker-compose.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '3.9' - -services: - repo: - image: dzikoysk/reposilite:3.0.0-alpha.23 - environment: - REPOSILITE_OPTS: "--token admin:secret --shared-configuration=/reposilite.cdn --shared-configuration-mode=copy --port=8080" - volumes: - - ./reposilite.cdn:/reposilite.cdn - - ../data:/app/data - ports: - - "9001:8080" - restart: unless-stopped - diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..c7963c4 --- /dev/null +++ b/run.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# Get the owner of the current folder (prevents permission problems when running with sudo) +user=$(stat -c "%u" .) +group=$(stat -c "%g" .) + +if [ ! -d ./data ]; then + mkdir ./data + chown "$user:$group" ./data +fi + +USER_ID=$user GROUP_ID=$group docker-compose up --build --exit-code-from maven-sync-job + +if [ $? -eq 0 ]; then + echo "Creating a tarball of the Releases" + tar zcvf ./maven-releases-packages.tar.gz ./data/repositories/releases/* 2>/dev/null + chown "$user:$group" ./maven-releases-packages.tar.gz + echo "Creating a tarball of the Snapshots" + tar zcvf ./maven-snapshots-packages.tar.gz ./data/repositories/snapshots/* 2>/dev/null + chown "$user:$group" ./maven-snapshots-packages.tar.gz + + echo "NEXT: mirror *.tar.gz and import into Artifactory." +else + echo "The sync failed, no tarballs created" + exit 255 +fi + +USER_ID=$user GROUP_ID=$group docker-compose down diff --git a/sync/.dockerignore b/sync/.dockerignore index 7c87d60..2b3a6a2 100644 --- a/sync/.dockerignore +++ b/sync/.dockerignore @@ -1,3 +1,4 @@ ** !settings.xml !resolve-deps-from-poms.sh +!wait-for-it.sh diff --git a/sync/Dockerfile b/sync/Dockerfile index ea74413..c483a41 100644 --- a/sync/Dockerfile +++ b/sync/Dockerfile @@ -2,8 +2,9 @@ FROM maven:alpine RUN mkdir /poms && mkdir -p /root/.m2 -ADD resolve-deps-from-poms.sh /resolve-deps-from-poms.sh -ADD settings.xml /root/.m2/settings.xml +COPY wait-for-it.sh /wait-for-it.sh +COPY resolve-deps-from-poms.sh /resolve-deps-from-poms.sh +COPY settings.xml /root/.m2/settings.xml VOLUME [ "/poms" ] diff --git a/sync/build-docker-image.sh b/sync/build-docker-image.sh deleted file mode 100755 index 738fd29..0000000 --- a/sync/build-docker-image.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -COCLI=docker # or podman - -docker build -t maven-sync-job . diff --git a/sync/resolve-deps-from-poms.sh b/sync/resolve-deps-from-poms.sh index 539a37d..d58d58a 100755 --- a/sync/resolve-deps-from-poms.sh +++ b/sync/resolve-deps-from-poms.sh @@ -1,6 +1,12 @@ #!/bin/sh -for pom in /poms/*; do - mvn -P central -f $pom dependency:go-offline; -done +/wait-for-it.sh -t 30 repo:80 +if [ $? -lt 30 ]; then + for pom in /poms/*; do + mvn -P central -f $pom dependency:go-offline; + done +else + echo "Can't connect to repository" + exit 255 +fi diff --git a/sync/settings.xml b/sync/settings.xml index 779b50c..a7179ab 100644 --- a/sync/settings.xml +++ b/sync/settings.xml @@ -3,12 +3,12 @@ central central - http://localhost:9001/releases + http://repo/releases confluent confluent - http://localhost:9001/confluent + http://repo/confluent diff --git a/sync/sync.sh b/sync/sync.sh deleted file mode 100755 index 2750dc0..0000000 --- a/sync/sync.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -COCLI=docker # or podman - -$COCLI run --network=host -it --rm -v $(pwd)/../examples:/poms maven-sync-job -cd ../data/repositories/releases/ -echo "Creating a tarball of the Releases" -tar zcvf ../../maven-releases-packages.tar.gz * 2>/dev/null -echo "Creating a tarball of the Snapshots" -cd ../snapshots -tar zcvf ../../maven-snapshots-packages.tar.gz * 2>/dev/null - -echo "NEXT: mirror *.tar.gz and import into Artifactory." - diff --git a/sync/wait-for-it.LICENCE b/sync/wait-for-it.LICENCE new file mode 100644 index 0000000..88d64e3 --- /dev/null +++ b/sync/wait-for-it.LICENCE @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright (c) 2016 Giles Hall + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/sync/wait-for-it.sh b/sync/wait-for-it.sh new file mode 100755 index 0000000..d990e0d --- /dev/null +++ b/sync/wait-for-it.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi