#!/bin/sh

check() {
    curl -s "$1" | grep -v '^\(#\|$\)' \
    | while read LINE
    do
        URL=$(printf "$LINE" | tr -s '\t' ' ' | cut -d" " -f2-);
        if curl -s --fail --connect-timeout 5 --max-time 5 "$URL" > /dev/null; then
            printf "$LINE","\n"
            printf "OK: %s\n" "$URL" > /dev/stderr
        else
            printf "ERROR (%i): %s\n" "$?" "$URL" > /dev/stderr
        fi
    done
}

# CONFIGURE BELOW
DIR="/srv/codevoid-gopher/tw/lists"

# check [URL] > $DIR/[FILENAME] &
check https://raw.githubusercontent.com/mdom/we-are-twtxt/master/we-are-bots.txt  > "$DIR/we-are-bots.txt" &
check https://raw.githubusercontent.com/mdom/we-are-twtxt/master/we-are-twtxt.txt > "$DIR/we-are-twtxt.txt" &
wait