tools-uclibc: pull out common code in run scripts and add parallel builds

5 files changed, 82 insertions, 236 deletions

diff --git a/tools-uclibc/common.sh b/tools-uclibc/common.sh
new file mode 100644
index 00000000..d1350d25
--- /dev/null
+++ b/tools-uclibc/common.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+source /etc/catalyst/catalyst.conf
+
+mydate=`date +%Y%m%d`
+
+
+undo_grsec() {
+  [[ -d /proc/sys/kernel/grsecurity ]] || return
+  for i in /proc/sys/kernel/grsecurity/chroot_* ; do
+    echo 0 > $i
+  done
+}
+
+
+banner() {
+cat << EOF | tee -a zzz.log > stage$1-$2-uclibc-$3.log
+
+************************************************************************
+*    stage$1-$2-uclibc-$3
+************************************************************************"
+
+EOF
+}
+
+
+do_stages() {
+  local arch=$1
+  local flavor=$2
+
+  for s in 1 2 3; do
+    local tgpath="${storedir}/builds/uclibc/${flavor}/${arch}"
+    local target="stage${s}-${arch}-uclibc-${flavor}-${mydate}.tar.bz2"
+    local tglink="stage${s}-${arch}-uclibc-${flavor}.tar.bz2"
+
+    if [[ ! -f "${tgpath}/${tglink}" ]]; then
+       touch stage${s}-${arch}-uclibc-${flavor}.log
+       echo "!!! ${tglink} at ${tgpath} doesn't exist" \
+         | tee -a zzz.log \
+         > stage${s}-${arch}-uclibc-${flavor}.err
+       return 1
+    fi
+
+    banner ${s} ${arch} ${flavor}
+    catalyst -f stage${s}-${arch}-uclibc-${flavor}.conf \
+      | tee -a zzz.log \
+      > stage${s}-${arch}-uclibc-${flavor}.log \
+      2> stage${s}-${arch}-uclibc-${flavor}.err
+
+    if [[ -f "${tgpath}/${target}" ]]; then
+      rm -f "${tgpath}/${tglink}"
+      ln -s ${target} "${tgpath}/${tglink}"
+    else
+      echo "!!! ${target} was not generated" \
+        | tee -a zzz.log \
+        >stage${s}-${arch}-uclibc-${flavor}.err
+      return 1
+    fi
+  done
+
+  return 0
+}
diff --git a/tools-uclibc/run-armv7a.sh b/tools-uclibc/run-armv7a.sh
index 7a248f5b..13b684f4 100755
--- a/tools-uclibc/run-armv7a.sh
+++ b/tools-uclibc/run-armv7a.sh
@@ -1,8 +1,6 @@
 #!/bin/bash
 
-source /etc/catalyst/catalyst.conf
-
-mydate=`date +%Y%m%d`
+source common.sh
 
 prepare_confs() {
   local arch=$1
@@ -42,54 +40,6 @@ prepare_confs() {
   sed -i "/^chost/d" stage3-${arch}-uclibc-${flavor}.conf
 }
 
-banner() {
-cat << EOF | tee -a zzz.log > stage$1-$2-uclibc-$3.log
-
-************************************************************************
-*    stage$1-$2-uclibc-$3
-************************************************************************"
-
-EOF
-}
-
-
-do_stages() {
-  local arch=$1
-  local flavor=$2
-
-  for s in 1 2 3; do
-    local tgpath="${storedir}/builds/uclibc/${flavor}/${arch}"
-    local target="stage${s}-${arch}-uclibc-${flavor}-${mydate}.tar.bz2"
-    local tglink="stage${s}-${arch}-uclibc-${flavor}.tar.bz2"
-
-    if [[ ! -f "${tgpath}/${tglink}" ]]; then
-       touch stage${s}-${arch}-uclibc-${flavor}.log
-       echo "!!! ${target} at ${tgpath} doesn't exit" \
-         | tee -a zzz.log \
-         > stage${s}-${arch}-uclibc-${flavor}.err
-       return 1
-    fi
-
-    banner ${s} ${arch} ${flavor}
-    catalyst -f stage${s}-${arch}-uclibc-${flavor}.conf \
-      | tee -a zzz.log \
-      > stage${s}-${arch}-uclibc-${flavor}.log \
-      2> stage${s}-${arch}-uclibc-${flavor}.err
-
-    if [[ -f "${tgpath}/${target}" ]]; then
-      rm -f "${tgpath}/${tglink}"
-      ln -s ${target} "${tgpath}/${tglink}"
-    else
-      echo "!!! ${target} was not generated" \
-        | tee -a zzz.log \
-        >stage${s}-${arch}-uclibc-${flavor}.err
-      return 1
-    fi
-  done
-
-  return 0
-}
-
 
 main() {
   >zzz.log
@@ -101,15 +51,12 @@ main() {
       prepare_confs ${arch} ${flavor}
     done
   done
-  
+
+  # No parallelization for arm.  Its too hard on the cpu!
   for arch in armv7a_hardfp armv7a; do
     for flavor in hardened vanilla; do
       do_stages ${arch} ${flavor}
-      ret=$?
-      if [[ $? == 1 ]]; then
-         echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
-         return 1
-      fi
+      [[ $? == 1 ]] && echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
     done
   done
 }
diff --git a/tools-uclibc/run-mips32r2.sh b/tools-uclibc/run-mips32r2.sh
index 8829b3dd..3655549e 100755
--- a/tools-uclibc/run-mips32r2.sh
+++ b/tools-uclibc/run-mips32r2.sh
@@ -1,8 +1,6 @@
 #!/bin/bash
 
-source /etc/catalyst/catalyst.conf
-
-mydate=`date +%Y%m%d`
+source common.sh
 
 prepare_confs() {
   local arch=$1
@@ -32,54 +30,6 @@ prepare_confs() {
   sed -i "/^chost/d" stage3-${arch}-uclibc-${flavor}.conf
 }
 
-banner() {
-cat << EOF | tee -a zzz.log > stage$1-$2-uclibc-$3.log
-
-************************************************************************
-*    stage$1-$2-uclibc-$3
-************************************************************************"
-
-EOF
-}
-
-
-do_stages() {
-  local arch=$1
-  local flavor=$2
-
-  for s in 1 2 3; do
-    local tgpath="${storedir}/builds/uclibc/${flavor}/${arch}"
-    local target="stage${s}-${arch}-uclibc-${flavor}-${mydate}.tar.bz2"
-    local tglink="stage${s}-${arch}-uclibc-${flavor}.tar.bz2"
-
-    if [[ ! -f "${tgpath}/${tglink}" ]]; then
-       touch stage${s}-${arch}-uclibc-${flavor}.log
-       echo "!!! ${target} at ${tgpath} doesn't exit" \
-         | tee -a zzz.log \
-         > stage${s}-${arch}-uclibc-${flavor}.err
-       return 1
-    fi
-
-    banner ${s} ${arch} ${flavor}
-    catalyst -f stage${s}-${arch}-uclibc-${flavor}.conf \
-      | tee -a zzz.log \
-      > stage${s}-${arch}-uclibc-${flavor}.log \
-      2> stage${s}-${arch}-uclibc-${flavor}.err
-
-    if [[ -f "${tgpath}/${target}" ]]; then
-      rm -f "${tgpath}/${tglink}"
-      ln -s ${target} "${tgpath}/${tglink}"
-    else
-      echo "!!! ${target} was not generated" \
-        | tee -a zzz.log \
-        >stage${s}-${arch}-uclibc-${flavor}.err
-      return 1
-    fi
-  done
-
-  return 0
-}
-
 
 main() {
   >zzz.log
@@ -91,15 +41,12 @@ main() {
       prepare_confs ${arch} ${flavor}
     done
   done
-  
+
+  # No parallelization for mips.  Its too hard on the cpu!
   for arch in mips32r2; do
     for flavor in hardened vanilla; do
       do_stages ${arch} ${flavor}
-      ret=$?
-      if [[ $? == 1 ]]; then
-         echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
-         return 1
-      fi
+      [[ $? == 1 ]] && echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
     done
   done
 }
diff --git a/tools-uclibc/run-mipsel3.sh b/tools-uclibc/run-mipsel3.sh
index 510639ca..4b70bed2 100755
--- a/tools-uclibc/run-mipsel3.sh
+++ b/tools-uclibc/run-mipsel3.sh
@@ -1,8 +1,6 @@
 #!/bin/bash
 
-source /etc/catalyst/catalyst.conf
-
-mydate=`date +%Y%m%d`
+source common.sh
 
 prepare_confs() {
   local arch=$1
@@ -32,54 +30,6 @@ prepare_confs() {
   sed -i "/^chost/d" stage3-${arch}-uclibc-${flavor}.conf
 }
 
-banner() {
-cat << EOF | tee -a zzz.log > stage$1-$2-uclibc-$3.log
-
-************************************************************************
-*    stage$1-$2-uclibc-$3
-************************************************************************"
-
-EOF
-}
-
-
-do_stages() {
-  local arch=$1
-  local flavor=$2
-
-  for s in 1 2 3; do
-    local tgpath="${storedir}/builds/uclibc/${flavor}/${arch}"
-    local target="stage${s}-${arch}-uclibc-${flavor}-${mydate}.tar.bz2"
-    local tglink="stage${s}-${arch}-uclibc-${flavor}.tar.bz2"
-
-    if [[ ! -f "${tgpath}/${tglink}" ]]; then
-       touch stage${s}-${arch}-uclibc-${flavor}.log
-       echo "!!! ${target} at ${tgpath} doesn't exit" \
-         | tee -a zzz.log \
-         > stage${s}-${arch}-uclibc-${flavor}.err
-       return 1
-    fi
-
-    banner ${s} ${arch} ${flavor}
-    catalyst -f stage${s}-${arch}-uclibc-${flavor}.conf \
-      | tee -a zzz.log \
-      > stage${s}-${arch}-uclibc-${flavor}.log \
-      2> stage${s}-${arch}-uclibc-${flavor}.err
-
-    if [[ -f "${tgpath}/${target}" ]]; then
-      rm -f "${tgpath}/${tglink}"
-      ln -s ${target} "${tgpath}/${tglink}"
-    else
-      echo "!!! ${target} was not generated" \
-        | tee -a zzz.log \
-        >stage${s}-${arch}-uclibc-${flavor}.err
-      return 1
-    fi
-  done
-
-  return 0
-}
-
 
 main() {
   >zzz.log
@@ -91,15 +41,12 @@ main() {
       prepare_confs ${arch} ${flavor}
     done
   done
-  
+
+  # No parallelization for mips.  Its too hard on the cpu!
   for arch in mipsel3; do
     for flavor in hardened vanilla; do
       do_stages ${arch} ${flavor}
-      ret=$?
-      if [[ $? == 1 ]]; then
-         echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
-         return 1
-      fi
+      [[ $? == 1 ]] && echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
     done
   done
 }
diff --git a/tools-uclibc/run.sh b/tools-uclibc/run.sh
index b1b8211c..ed0a14fc 100755
--- a/tools-uclibc/run.sh
+++ b/tools-uclibc/run.sh
@@ -1,15 +1,6 @@
 #!/bin/bash
 
-source /etc/catalyst/catalyst.conf
-
-mydate=`date +%Y%m%d`
-
-undo_grsec() {
-  [[ -d /proc/sys/kernel/grsecurity ]] || return
-  for i in /proc/sys/kernel/grsecurity/chroot_* ; do
-    echo 0 > $i
-  done
-}
+source common.sh
 
 prepare_confs() {
   local arch=$1
@@ -43,54 +34,6 @@ prepare_confs() {
   sed -i "/^chost/d" stage3-${arch}-uclibc-${flavor}.conf
 }
 
-banner() {
-cat << EOF | tee -a zzz.log > stage$1-$2-uclibc-$3.log
-
-************************************************************************
-*    stage$1-$2-uclibc-$3
-************************************************************************"
-
-EOF
-}
-
-
-do_stages() {
-  local arch=$1
-  local flavor=$2
-
-  for s in 1 2 3; do
-    local tgpath="${storedir}/builds/uclibc/${flavor}/${arch}"
-    local target="stage${s}-${arch}-uclibc-${flavor}-${mydate}.tar.bz2"
-    local tglink="stage${s}-${arch}-uclibc-${flavor}.tar.bz2"
-
-    if [[ ! -f "${tgpath}/${tglink}" ]]; then
-       touch stage${s}-${arch}-uclibc-${flavor}.log
-       echo "!!! ${target} at ${tgpath} doesn't exit" \
-         | tee -a zzz.log \
-         > stage${s}-${arch}-uclibc-${flavor}.err
-       return 1
-    fi
-
-    banner ${s} ${arch} ${flavor}
-    catalyst -f stage${s}-${arch}-uclibc-${flavor}.conf \
-      | tee -a zzz.log \
-      > stage${s}-${arch}-uclibc-${flavor}.log \
-      2> stage${s}-${arch}-uclibc-${flavor}.err
-
-    if [[ -f "${tgpath}/${target}" ]]; then
-      rm -f "${tgpath}/${tglink}"
-      ln -s ${target} "${tgpath}/${tglink}"
-    else
-      echo "!!! ${target} was not generated" \
-        | tee -a zzz.log \
-        >stage${s}-${arch}-uclibc-${flavor}.err
-      return 1
-    fi
-  done
-
-  return 0
-}
-
 
 main() {
   >zzz.log
@@ -104,15 +47,15 @@ main() {
       prepare_confs ${arch} ${flavor}
     done
   done
-  
+
+  # The parallelization `( do_stages ... ) &` doesn't work here
+  # if catalyst is using snapcache, bug #519656
   for arch in amd64 i686; do
     for flavor in hardened vanilla; do
-      do_stages ${arch} ${flavor}
-      ret=$?
-      if [[ $? == 1 ]]; then
-         echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
-         return 1
-      fi
+      (
+        do_stages ${arch} ${flavor}
+        [[ $? == 1 ]] && echo "FAILURE at ${arch} ${flavor} " | tee zzz.log
+      ) &
     done
   done
 }