Added support for Debian and ARM (armel, armhf, arm64).HEADmaster

Removed the unused tmp/ directory.
Output format of dump/find/identify has changed.
The folder structure of the database has changed.

6 files changed, 177 insertions, 54 deletions

diff --git a/common/libc.sh b/common/libc.sh
index 7e2a760..2135b98 100644
--- a/common/libc.sh
+++ b/common/libc.sh
@@ -16,20 +16,35 @@ extract_label() {
 }
 
 dump_libc_start_main_ret() {
-  local call_main=`objdump -D $1 \
-    | egrep -A 100 '<__libc_start_main.*>' \
-    | grep call \
-    | egrep -B 1 '<exit.*>' \
-    | head -n 1 \
-    | extract_label`
-  local offset=`objdump -D $1 | egrep -A 1 "(^| )$call_main:" | tail -n 1 | extract_label`
+  if [ `echo "$2" | egrep "i386|amd64" - ` ] \
+    && [ -x "$(command -v objdump)" ]; then
+    local call_main=`objdump -D $1 \
+      | egrep -A 100 '<__libc_start_main.*>' \
+      | grep call \
+      | egrep -B 1 '<exit.*>' \
+      | head -n 1 \
+      | extract_label`
+    local offset=`objdump -D $1 | egrep -A 1 "(^| )$call_main:" | tail -n 1 | extract_label`
+  elif [ `echo "$2" | egrep "armel|armhf|arm64" - ` ] \
+    && [ -x "$(command -v aarch64-linux-gnu-objdump)" ]; then
+    local call_main=`aarch64-linux-gnu-objdump -D $1 \
+      | egrep -A 100 '<__libc_start_main.*>' \
+      | grep -Pe "(bl\t|blr\t|blx\t|bx\t)" \
+      | egrep -B 1 '<exit.*>' \
+      | head -n 1 \
+      | extract_label`
+    local offset=`aarch64-linux-gnu-objdump -D $1 \
+      | egrep -A 1 "(^| )$call_main:" \
+      | tail -n 1 \
+      | extract_label`
+  fi
   if [[ "$offset" != "" ]]; then
     echo "__libc_start_main_ret $offset"
   fi
 }
 
 dump_bin_sh() {
-  local offset=`strings -a -t x $1 | grep '/bin/sh' | extract_label`
+  local offset=`strings -a -t x $1 | grep '/bin/sh' | head -n 1 | extract_label`
   if [[ "$offset" != "" ]]; then
     echo "str_bin_sh $offset"
   fi
@@ -39,35 +54,48 @@ process_libc() {
   local libc=$1
   local id=$2
   local info=$3
-  echo "  -> Writing libc to db/${id}.so"
-  cp $libc db/${id}.so
-  echo "  -> Writing symbols to db/${id}.symbols"
-  (dump_symbols $libc; dump_libc_start_main_ret $libc; dump_bin_sh $libc) \
-     > db/${id}.symbols
-  echo "  -> Writing version info"
-  echo "$info" > db/${id}.info
+  local sha256=$4
+  [[ -d db/$info ]] || mkdir db/$info
+  echo "  -> Writing libc to db/${info}/${id}.so"
+  cp $libc db/${info}/${id}.so
+  echo "  -> Writing symbols to db/${info}/${id}.symbols"
+  (dump_symbols $libc; dump_libc_start_main_ret $libc $id; dump_bin_sh $libc) \
+     > db/${info}/${id}.symbols
+  echo "  -> Writing SHA-256 to db/${info}/${id}.sha256"
+  echo "$sha256" > db/${info}/${id}.sha256
 }
 
 check_id() {
   local id=$1
-  if [[ -e db/${id}.info ]]; then
-    echo "  -> Already have this version, 'rm db/${id}.*' to force"
+  local info=$2
+  if [[ -e db/${info}/${id}.sha256 ]]; then
+    echo "  -> Already have this version, 'rm db/${info}/${id}.*' to force"
+    return 1
+  fi
+  return 0
+}
+
+check_sha256() {
+  local sha256=$1
+  local libc=`grep -r $sha256 db/ 2>/dev/null | cut -d ':' -f 1 - 2>/dev/null | sed 's/\.sha256/\.so/gI'`
+  if [[ "$libc" != "" ]]; then
+    echo "  -> Found SHA-256 match: $libc"
     return 1
   fi
   return 0
 }
 
-# ===== Ubuntu ===== #
+# ===== Debian and Ubuntu ===== #
 
-get_ubuntu() {
+get_deb() {
   local url="$1"
   local info="$2"
   local tmp=`mktemp -d || mktemp -d -t "libc-database" || die "Cannot get temp dir"`
   echo "Getting $info"
   echo "  -> Location: $url"
-  local id=`echo $url | perl -n -e '/(libc6[^\/]*)\./ && print $1'`
+  local id=`echo $url | perl -n -e '/(libc6[^\/]*)\./ && print $1' | sed 's/\%2b/+/gI'`
   echo "  -> ID: $id"
-  check_id $id || return
+  check_id $id $info || return
   echo "  -> Downloading package"
   wget $url 2>/dev/null -O $tmp/pkg.deb || die "Failed to download package from $url"
   echo "  -> Extracting package"
@@ -76,7 +104,13 @@ get_ubuntu() {
   tar xf data.tar.* || die "tar failed"
   popd 1>/dev/null
   local libc=`find $tmp -name libc.so.6 || die "Cannot locate libc.so.6"`
-  process_libc $libc $id $info
+  if [[ ! -e $libc ]]; then 
+    echo "  -> The package does not seem to contain a valid libc (e.g. just contains a broken symbolic link)"
+    return
+  fi
+  local sha256=`sha256sum $libc`
+  check_sha256 $sha256 || return
+  process_libc $libc $id $info $sha256
   rm -rf $tmp
 }
 
@@ -88,14 +122,22 @@ get_current_ubuntu() {
   echo "Getting package location for ubuntu-$version-$arch"
   local url=`(wget http://packages.ubuntu.com/$version/$arch/$pkg/download -O - 2>/dev/null \
                | grep -oh 'http://[^"]*libc6[^"]*.deb') || die "Failed to get package version"`
-  get_ubuntu $url $info
+  get_deb $url $info
+}
+
+get_all_debian() {
+  local info=debian-$1
+  local url=$2
+  for f in `wget $url/ -O - 2>/dev/null | egrep -oh 'libc6(-i386|-i686|-amd64|-armel|-armhf|-arm64)?(-cross)?_[^"]*(i386|amd64|armel|armhf|arm64|all)\.deb' | grep -v "</a>"`; do
+    get_deb $url/$f $info
+  done
 }
 
 get_all_ubuntu() {
-  local info=$1
+  local info=ubuntu-$1
   local url=$2
-  for f in `wget $url/ -O - 2>/dev/null | egrep -oh 'libc6(-i386|-amd64)?_[^"]*' |grep -v "</a>"`; do
-    get_ubuntu $url/$f $1
+  for f in `wget $url/ -O - 2>/dev/null | egrep -oh 'libc6(-i386|-i686|-amd64|-armel|-armel-armhf|-armhf|-arm64)?(-cross)?_[^"]*(i386|amd64|armel|armhf|arm64|all)\.deb' | grep -v "</a>"`; do
+    get_deb $url/$f $info
   done
 }
 
@@ -105,8 +147,10 @@ add_local() {
   local libc=$1
   [[ -e $libc ]] || return
   local info="local"
-  local id="local-`sha1sum $libc`"
+  local id="local-`sha256sum $libc`"
   echo "Adding local libc $libc (id $id)"
-  check_id $id || return
-  process_libc $libc $id $info
+  check_id $id $info || return
+  local sha256=`sha256sum $libc`
+  check_sha256 $sha256 || return
+  process_libc $libc $id $info $sha256
 }
diff --git a/dump b/dump
index b45dd7e..d76eb53 100755
--- a/dump
+++ b/dump
@@ -10,7 +10,12 @@ if [[ $# == 0 ]]; then
 else
   names="$@"
 fi
-for name in $names; do
-  offset=`cat db/${id}.symbols | grep "^$name " | cut -d' ' -f2`
-  echo "offset_${name} = 0x${offset}"
+for info in db/*; do
+  if [[ -d $info ]] && [[ -e $info/$id.symbols ]]; then
+    echo "${info}/${id}.symbols:"
+    for name in $names; do
+      offset=`cat ${info}/${id}.symbols | grep "^$name " | cut -d' ' -f2`
+      echo "offset_${name} = 0x${offset}"
+    done
+  fi
 done
diff --git a/find b/find
index 0039a12..dda3fc3 100755
--- a/find
+++ b/find
@@ -8,7 +8,7 @@ function find_single() {
   name=$1
   address=$2
   addr_last12=`echo -n "$address" | tail -c 3`
-  grep -i -e "^$name .*$addr_last12$" db/*.symbols \
+  grep -r --include=\*.symbols -i -e "^$name .*$addr_last12$" db/ \
     | perl -n -e '/db\/(.*)\.symbols/ && print "$1\n"' \
     | sort
 }
@@ -27,5 +27,5 @@ function find() {
 }
 
 for id in `find "$@"`; do
-  echo "`cat db/${id}.info` (id $id)"
+  echo "db/${id}.so"
 done
diff --git a/get b/get
index 90bdcb1..59e8655 100755
--- a/get
+++ b/get
@@ -1,9 +1,9 @@
 #!/bin/bash
 . common/libc.sh
 
-get_current_ubuntu trusty i386 libc6
-get_current_ubuntu trusty amd64 libc6
-get_current_ubuntu trusty amd64 libc6-i386
+#get_current_ubuntu trusty i386 libc6
+#get_current_ubuntu trusty amd64 libc6
+#get_current_ubuntu trusty amd64 libc6-i386
 #get_current_ubuntu utopic i386 libc6
 #get_current_ubuntu utopic amd64 libc6
 #get_current_ubuntu utopic amd64 libc6-i386
@@ -13,18 +13,90 @@ get_current_ubuntu trusty amd64 libc6-i386
 #get_current_ubuntu wily i386 libc6
 #get_current_ubuntu wily amd64 libc6
 #get_current_ubuntu wily amd64 libc6-i386
-get_current_ubuntu xenial i386 libc6
-get_current_ubuntu xenial amd64 libc6
-get_current_ubuntu xenial amd64 libc6-i386
+#get_current_ubuntu xenial i386 libc6
+#get_current_ubuntu xenial amd64 libc6
+#get_current_ubuntu xenial amd64 libc6-i386
 #get_current_ubuntu yakkety i386 libc6
 #get_current_ubuntu yakkety amd64 libc6
 #get_current_ubuntu yakkety amd64 libc6-i386
-get_current_ubuntu zesty i386 libc6
-get_current_ubuntu zesty amd64 libc6
-get_current_ubuntu zesty amd64 libc6-i386
-get_current_ubuntu artful i386 libc6
-get_current_ubuntu artful amd64 libc6
-get_current_ubuntu artful amd64 libc6-i386
-
-get_all_ubuntu archive-eglibc http://security.ubuntu.com/ubuntu/pool/main/e/eglibc/
-get_all_ubuntu archive-glibc http://security.ubuntu.com/ubuntu/pool/main/g/glibc/
+#get_current_ubuntu zesty i386 libc6
+#get_current_ubuntu zesty amd64 libc6
+#get_current_ubuntu zesty amd64 libc6-i386
+#get_current_ubuntu artful i386 libc6
+#get_current_ubuntu artful amd64 libc6
+#get_current_ubuntu artful amd64 libc6-i386
+
+get_all_debian main-cross-toolchain-base http://http.us.debian.org/debian/pool/main/c/cross-toolchain-base/
+get_all_debian main-cross-toolchain-base-ports http://http.us.debian.org/debian/pool/main/c/cross-toolchain-base-ports/
+get_all_debian main-eglibc http://http.us.debian.org/debian/pool/main/e/eglibc/
+get_all_debian main-glibc http://http.us.debian.org/debian/pool/main/g/glibc/
+
+get_all_debian archive-eglibc http://archive.debian.org/debian/pool/main/e/eglibc/
+get_all_debian archive-glibc http://archive.debian.org/debian/pool/main/g/glibc/
+
+get_all_debian mirrordirector-raspbian-cross-toolchain-base http://mirrordirector.raspbian.org/raspbian/pool/main/c/cross-toolchain-base/
+get_all_debian mirrordirector-raspbian-cross-toolchain-base-ports http://mirrordirector.raspbian.org/raspbian/pool/main/c/cross-toolchain-base-ports/
+get_all_debian mirrordirector-raspbian-eglibc http://mirrordirector.raspbian.org/raspbian/pool/main/e/eglibc/
+get_all_debian mirrordirector-raspbian-glibc http://mirrordirector.raspbian.org/raspbian/pool/main/g/glibc/
+
+get_all_debian archive-raspbian-cross-toolchain-base https://archive.raspbian.org/raspbian/pool/main/c/cross-toolchain-base/
+get_all_debian archive-raspbian-cross-toolchain-base-ports https://archive.raspbian.org/raspbian/pool/main/c/cross-toolchain-base-ports/
+get_all_debian archive-raspbian-eglibc https://archive.raspbian.org/raspbian/pool/main/e/eglibc/
+get_all_debian archive-raspbian-glibc https://archive.raspbian.org/raspbian/pool/main/g/glibc/
+
+get_all_ubuntu main-arm64-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/a/arm64-cross-toolchain-base/
+get_all_ubuntu main-armel-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/a/armel-cross-toolchain-base/
+get_all_ubuntu main-armhf-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/a/armhf-cross-toolchain-base/
+get_all_ubuntu main-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/c/cross-toolchain-base/
+get_all_ubuntu main-cross-toolchain-base-ports http://security.ubuntu.com/ubuntu/pool/main/c/cross-toolchain-base-ports/
+get_all_ubuntu main-eglibc http://security.ubuntu.com/ubuntu/pool/main/e/eglibc/
+get_all_ubuntu main-glibc http://security.ubuntu.com/ubuntu/pool/main/g/glibc/
+#get_all_ubuntu main-powerpc-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu main-powerpc-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/main/p/ppc64el-cross-toolchain-base/
+get_all_ubuntu universe-arm64-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/a/arm64-cross-toolchain-base/
+get_all_ubuntu universe-armel-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/a/armel-cross-toolchain-base/
+get_all_ubuntu universe-armhf-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/a/armhf-cross-toolchain-base/
+get_all_ubuntu universe-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base/
+get_all_ubuntu universe-cross-toolchain-base-ports http://security.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base-ports/
+get_all_ubuntu universe-eglibc http://security.ubuntu.com/ubuntu/pool/universe/e/eglibc/
+get_all_ubuntu universe-glibc http://security.ubuntu.com/ubuntu/pool/universe/g/glibc/
+#get_all_ubuntu universe-powerpc-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu universe-powerpc-cross-toolchain-base http://security.ubuntu.com/ubuntu/pool/universe/p/ppc64el-cross-toolchain-base/
+
+get_all_ubuntu old-main-arm64-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/a/arm64-cross-toolchain-base/
+get_all_ubuntu old-main-armel-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/a/armel-cross-toolchain-base/
+get_all_ubuntu old-main-armhf-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/a/armhf-cross-toolchain-base/
+get_all_ubuntu old-main-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/c/cross-toolchain-base/
+get_all_ubuntu old-main-cross-toolchain-base-ports http://old-releases.ubuntu.com/ubuntu/pool/main/c/cross-toolchain-base-ports/
+get_all_ubuntu old-main-eglibc http://old-releases.ubuntu.com/ubuntu/pool/main/e/eglibc/
+get_all_ubuntu old-main-glibc http://old-releases.ubuntu.com/ubuntu/pool/main/g/glibc/
+#get_all_ubuntu old-main-powerpc-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu old-main-powerpc-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/main/p/ppc64el-cross-toolchain-base/
+get_all_ubuntu old-universe-arm64-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/a/arm64-cross-toolchain-base/
+get_all_ubuntu old-universe-armel-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/a/armel-cross-toolchain-base/
+get_all_ubuntu old-universe-armhf-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/a/armhf-cross-toolchain-base/
+get_all_ubuntu old-universe-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base/
+get_all_ubuntu old-universe-cross-toolchain-base-ports http://old-releases.ubuntu.com/ubuntu/pool/universe/c/cross-toolchain-base-ports/
+get_all_ubuntu old-universe-eglibc http://old-releases.ubuntu.com/ubuntu/pool/universe/e/eglibc/
+get_all_ubuntu old-universe-glibc http://old-releases.ubuntu.com/ubuntu/pool/universe/g/glibc/
+#get_all_ubuntu old-universe-powerpc-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu old-universe-powerpc-cross-toolchain-base http://old-releases.ubuntu.com/ubuntu/pool/universe/p/ppc64el-cross-toolchain-base/
+
+get_all_ubuntu ports-main-arm64-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/a/arm64-cross-toolchain-base/
+get_all_ubuntu ports-main-armel-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/a/armel-cross-toolchain-base/
+get_all_ubuntu ports-main-armhf-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/a/armhf-cross-toolchain-base/
+get_all_ubuntu ports-main-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/c/cross-toolchain-base/
+get_all_ubuntu ports-main-cross-toolchain-base-ports http://ports.ubuntu.com/ubuntu-ports/pool/main/c/cross-toolchain-base-ports/
+get_all_ubuntu ports-main-eglibc http://ports.ubuntu.com/ubuntu-ports/pool/main/e/eglibc/
+get_all_ubuntu ports-main-glibc http://ports.ubuntu.com/ubuntu-ports/pool/main/g/glibc/
+#get_all_ubuntu ports-main-powerpc-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu ports-main-powerpc-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/main/p/ppc64el-cross-toolchain-base/
+get_all_ubuntu ports-universe-arm64-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/a/arm64-cross-toolchain-base/
+get_all_ubuntu ports-universe-armel-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/a/armel-cross-toolchain-base/
+get_all_ubuntu ports-universe-armhf-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/a/armhf-cross-toolchain-base/
+get_all_ubuntu ports-universe-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/c/cross-toolchain-base/
+get_all_ubuntu ports-universe-cross-toolchain-base-ports http://ports.ubuntu.com/ubuntu-ports/pool/universe/c/cross-toolchain-base-ports/
+get_all_ubuntu ports-universe-eglibc http://ports.ubuntu.com/ubuntu-ports/pool/universe/e/eglibc/
+get_all_ubuntu ports-universe-glibc http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/glibc/
+#get_all_ubuntu ports-universe-powerpc-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/p/powerpc-cross-toolchain-base/
+#get_all_ubuntu ports-universe-powerpc-cross-toolchain-base http://ports.ubuntu.com/ubuntu-ports/pool/universe/p/ppc64el-cross-toolchain-base/
diff --git a/identify b/identify
index 2e96fd9..6d603cf 100755
--- a/identify
+++ b/identify
@@ -3,6 +3,10 @@ if [[ $# != 1 ]]; then
   echo >&2 "Usage: $0 path/to/libc.so"
   exit 2
 fi
-libc=$1
-sha1=`sha1sum $libc | awk '{print $1}'`
-sha1sum db/*.so | grep "$sha1 " | perl -n -e '/db\/(.*)\.so/&&print "id $1\n"'
+
+. common/libc.sh
+
+[[ -e $1 ]] || exit
+sha256=`sha256sum $1`
+check_sha256 $sha256 || exit
+echo "No SHA-256 match found in the database"
diff --git a/tmp/.gitignore b/tmp/.gitignore
deleted file mode 100644
index d6b7ef3..0000000
--- a/tmp/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore