make_source_tarball.sh: Make compression threads and fetch jobs configurable

This should improve speed a lot:
- Increasing the number of fetch jobs speeds up a lot the
  downloads when resuming from a previously interrupted
  download session as checking for new commits can have
  a big latency, so checking many as repositories as
  possible at the same time speeds up things a lot.
  With Replicant 4.2 0004, using "-j 100" worked well for
  me, however it didn't seem to have reached 100 parallel
  download jobs, so using more would not improve
  performance in my case.
- As for the number of compression threads, one has to consider
  it carefully: xz -9e uses quite a big amount of RAM when compressing,
  so using too much threads can result in xz being killed because
  it used too much RAM. In another hand using 1 thread per core
  seems to divide the amount of time spent compressing by the number
  of threads.

Signed-off-by: Denis 'GNUtoo' Carikli <GNUtoo@cyberdimension.org>

1 files changed, 38 insertions, 7 deletions

diff --git a/make_source_tarball.sh b/make_source_tarball.sh
index 41b8195..c07bb56 100755
--- a/make_source_tarball.sh
+++ b/make_source_tarball.sh
@@ -19,10 +19,45 @@ set -e
 
 usage()
 {
-	echo "$0 <tag> <output directory>"
+	echo "$0 <tag> <output directory> [-j N] [-T N]"
+	echo "  -j N        use at most N download jobs to run in parallel;"
+	echo "              the default is 4;"
+	echo "  -T N        use at most N threads to compress the tarball;"
+	echo "              the default is 1; set to 0 to use as many threads"
+	echo "              as there are processor cores"
+
 	exit 1
 }
 
+jobs=4
+threads=1
+
+get_option()
+{
+	option="$1"
+	value="$2"
+
+	if [ "${option}" = "-T" ] ; then
+		threads="${value}"
+	elif [ "${option}" = "-j" ] ; then
+		jobs="${value}"
+	else
+		usage
+	fi
+}
+
+if [ $# -ne 2 -a $# -ne 4 -a $# -ne 6 ] ; then
+	usage
+elif [ $# -eq 4 ] ; then
+	get_option "$3" "$4"
+elif [ $# -eq 6 ] ; then
+	if [ "$3" = "$5" ] ; then
+		usage
+	fi
+	get_option "$3" "$4"
+	get_option "$5" "$6"
+fi
+
 tag="$1"
 outdir="$2"
 
@@ -33,10 +68,6 @@ version="$(echo ${tag} | sed 's/replicant-//')"
 major_version="$(echo ${version} | sed 's/-.*//')"
 minor_version="$(echo ${version} | sed 's/.*-//')"
 
-if [ $# -ne 2 ] ; then
-	usage
-fi
-
 mkdir -p "${replicant_dir}"
 cd "${replicant_dir}"
 if [ ! -d .repo ] ; then
@@ -57,7 +88,7 @@ if [ $(echo ${major_version} | sed 's/\..*//') -lt 6 ] ; then
 	    .repo/manifests/default.xml
 fi
 
-repo sync
+repo sync -j "${jobs}"
 
 if [ $(echo $major_version | sed 's/\..*//') -lt 6 ] ; then
 	git -C .repo/manifests reset --hard
@@ -71,4 +102,4 @@ fi
 
 # TODO: Make the tarball reproducible by removing metadata
 
-xz -9e --verbose "${tarball}"
+xz -T "${threads}" -9e --verbose "${tarball}"