A bash script who parse access log and create daily stats. Require borg and goaccess
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

172 lines
5.1 KiB

#!/bin/bash
## author : Dryusdan
## date : 22/10/2019
## description : A goaccess stat generator
## usage : ./create_stats.sh {daily|weekly|monthly|yearly}
## Bash strict mode ####################################
set -o errexit # abort on nonzero exitstatus
set -o nounset # abort on unbound variable
set -o pipefail # don't hide errors within pipes
## Bash color ##########################################
# Set colors
RED='\033[0;31m'
GREEN='\033[00;32m'
YELLOW='\033[00;33m'
BLUE='\033[00;34m'
PURPLE='\033[00;35m'
CYAN='\033[00;36m'
LIGHTGRAY='\033[00;37m'
LRED='\033[01;31m'
LGREEN='\033[01;32m'
LYELLOW='\033[01;33m'
LBLUE='\033[01;34m'
LPURPLE='\033[01;35m'
LCYAN='\033[01;36m'
WHITE='\033[01;37m'
NC='\033[0m' # No Color
## Logs ################################################
readonly SCRIPTNAME="$(basename "$0")"
info() { echo -e "${LBLUE}[INFO] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
warning() { echo -e "${YELLOW}[WARNING] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
error() { echo -e "${LRED}[ERROR] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
fatal() { echo -e "${RED}[FATAL] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; exit 1 ; }
########################################################
## Define variables ###################################
info "Define variables"
DATE=$(date '+%Y-%m-%d')
REPONAME="rp-1"
MOUNTPATH="/mnt"
TMP_LOG_PATH="/data/tmp/logs"
HTML_FILE="/var/www/stats_dryusdan_fr/www"
HTML_DAY_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/daily/%B/%d')"
HTML_WEEK_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/weekly/%V')"
HTML_MONTH_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/monthly/%B')"
HTML_YEAR_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/yearly')"
info "Mount backup"
/usr/bin/borg mount /data/borg/${REPONAME}/${REPONAME} ${MOUNTPATH}
LOG_PATH="$(ls -td /mnt/*/ | head -1)/var/log/archive-nginx"
info "Create temp folder if not exist"
mkdir -p "${TMP_LOG_PATH}"
if [ -d "${TMP_LOG_PATH}/nginx" ]; then
info "Remove ${TMP_LOG_PATH}/archive-nginx"
rm -rf "${TMP_LOG_PATH}"
mkdir -p "${TMP_LOG_PATH}"
fi
usage () {
echo "Usage: ${SCRIPTNAME} {daily|weekly|monthly|yearly}"
info "umount backup"
/usr/bin/borg umount ${MOUNTPATH}
fatal "Bad parameter ${0}"
}
daily (){
info "Daily stat"
info "Copy backup"
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
info "Create final folder"
mkdir -p ${HTML_DAY_FILE}
cd ${TMP_LOG_PATH}
for file in $(ls ${TMP_LOG_PATH})
do
info "Generate stat for ${file}"
domain=$(echo ${file} | sed "s/\.access\.log.*//g")
goaccess --output=${HTML_DAY_FILE}/${domain}-$(date --date='1 days ago' '+%Y-%m-%d').html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}
done
cd /
rm -rf ${TMP_LOG_PATH}
}
weekly (){
info "Weekly stat"
info "Copy backup"
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
cp -r "${LOG_PATH}/"*access*.{2,3,4,5,6,7}.gz "${TMP_LOG_PATH}"
info "uncompress everything"
gzip -d ${TMP_LOG_PATH}/*.gz
mkdir -p ${HTML_WEEK_FILE}
cd ${TMP_LOG_PATH}
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
do
info "Generate stat for ${file}"
if [ $(ls ${file}".access.log."* | wc -l ) -eq 7 ]; then
goaccess --output=${HTML_WEEK_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1,2,3,4,5,6,7}
fi
done
cd /
rm -rf $TMP_LOG_PATH
}
monthly () {
info "Monthly stat"
DAY_IN_MONTH=$(cal $(date --date='1 days ago' +"%m %Y") | awk 'NF {DAYS = $NF}; END {print DAYS}')
info "Copy backup"
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
cd "${LOG_PATH}"
eval cp -r *access.log.{2..${DAY_IN_MONTH}}.gz "${TMP_LOG_PATH}"
info "uncompress everything"
gzip -d ${TMP_LOG_PATH}/*.gz
mkdir -p ${HTML_MONTH_FILE}
cd ${TMP_LOG_PATH}
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
do
info "Generate stat for ${file}"
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
eval goaccess --output=${HTML_MONTH_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
done
cd /
rm -rf ${TMP_LOG_PATH}
}
yearly () {
info "Yearly stat"
DAY_IN_YEAR=$(date --date='1 day ago' +"%j")
info "Copy backup"
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
cd "${LOG_PATH}"
eval cp -r *access.log.{2..${DAY_IN_YEAR}}.gz "${TMP_LOG_PATH}"
info "uncompress everything"
gzip -d ${TMP_LOG_PATH}/*.gz
mkdir -p ${HTML_YEAR_FILE}
cd ${TMP_LOG_PATH}
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
do
info "Generate stat for ${file}"
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
eval goaccess --output=${HTML_YEAR_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
done
cd /
rm -rf ${TMP_LOG_PATH}
}
case "${1}" in
daily)
daily
;;
weekly)
weekly
;;
monthly)
monthly
;;
yearly)
yearly
;;
*)
usage
esac
info "umount backup"
/usr/bin/borg umount ${MOUNTPATH}