173 lines
5.1 KiB
Bash
173 lines
5.1 KiB
Bash
#!/bin/bash
|
|
## author : Dryusdan
|
|
## date : 22/10/2019
|
|
## description : A goaccess stat generator
|
|
## usage : ./create_stats.sh {daily|weekly|monthly|yearly}
|
|
|
|
## Bash strict mode ####################################
|
|
set -o errexit # abort on nonzero exitstatus
|
|
set -o nounset # abort on unbound variable
|
|
set -o pipefail # don't hide errors within pipes
|
|
|
|
## Bash color ##########################################
|
|
# Set colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[00;32m'
|
|
YELLOW='\033[00;33m'
|
|
BLUE='\033[00;34m'
|
|
PURPLE='\033[00;35m'
|
|
CYAN='\033[00;36m'
|
|
LIGHTGRAY='\033[00;37m'
|
|
LRED='\033[01;31m'
|
|
LGREEN='\033[01;32m'
|
|
LYELLOW='\033[01;33m'
|
|
LBLUE='\033[01;34m'
|
|
LPURPLE='\033[01;35m'
|
|
LCYAN='\033[01;36m'
|
|
WHITE='\033[01;37m'
|
|
NC='\033[0m' # No Color
|
|
|
|
## Logs ################################################
|
|
readonly SCRIPTNAME="$(basename "$0")"
|
|
info() { echo -e "${LBLUE}[INFO] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
|
warning() { echo -e "${YELLOW}[WARNING] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
|
error() { echo -e "${LRED}[ERROR] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
|
fatal() { echo -e "${RED}[FATAL] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; exit 1 ; }
|
|
########################################################
|
|
|
|
## Define variables ###################################
|
|
info "Define variables"
|
|
DATE=$(date '+%Y-%m-%d')
|
|
REPONAME="rp-1"
|
|
MOUNTPATH="/mnt"
|
|
TMP_LOG_PATH="/data/tmp/logs"
|
|
HTML_FILE="/var/www/stats_dryusdan_fr/www"
|
|
HTML_DAY_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/daily/%B/%d')"
|
|
HTML_WEEK_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/weekly/%V')"
|
|
HTML_MONTH_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/monthly/%B')"
|
|
HTML_YEAR_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/yearly')"
|
|
|
|
|
|
info "Mount backup"
|
|
/usr/bin/borg mount /data/borg/${REPONAME}/${REPONAME} ${MOUNTPATH}
|
|
LOG_PATH="$(ls -td /mnt/*/ | head -1)/var/log/archive-nginx"
|
|
|
|
info "Create temp folder if not exist"
|
|
mkdir -p "${TMP_LOG_PATH}"
|
|
|
|
if [ -d "${TMP_LOG_PATH}/nginx" ]; then
|
|
info "Remove ${TMP_LOG_PATH}/archive-nginx"
|
|
rm -rf "${TMP_LOG_PATH}"
|
|
mkdir -p "${TMP_LOG_PATH}"
|
|
fi
|
|
|
|
usage () {
|
|
echo "Usage: ${SCRIPTNAME} {daily|weekly|monthly|yearly}"
|
|
info "umount backup"
|
|
/usr/bin/borg umount ${MOUNTPATH}
|
|
fatal "Bad parameter ${0}"
|
|
}
|
|
|
|
daily (){
|
|
info "Daily stat"
|
|
info "Copy backup"
|
|
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
|
info "Create final folder"
|
|
mkdir -p ${HTML_DAY_FILE}
|
|
cd ${TMP_LOG_PATH}
|
|
for file in $(ls ${TMP_LOG_PATH})
|
|
do
|
|
info "Generate stat for ${file}"
|
|
domain=$(echo ${file} | sed "s/\.access\.log.*//g")
|
|
goaccess --output=${HTML_DAY_FILE}/${domain}-$(date --date='1 days ago' '+%Y-%m-%d').html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}
|
|
done
|
|
cd /
|
|
rm -rf ${TMP_LOG_PATH}
|
|
|
|
}
|
|
|
|
weekly (){
|
|
info "Weekly stat"
|
|
info "Copy backup"
|
|
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
|
cp -r "${LOG_PATH}/"*access*.{2,3,4,5,6,7}.gz "${TMP_LOG_PATH}"
|
|
info "uncompress everything"
|
|
gzip -d ${TMP_LOG_PATH}/*.gz
|
|
mkdir -p ${HTML_WEEK_FILE}
|
|
cd ${TMP_LOG_PATH}
|
|
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
|
do
|
|
info "Generate stat for ${file}"
|
|
if [ $(ls ${file}".access.log."* | wc -l ) -eq 7 ]; then
|
|
goaccess --output=${HTML_WEEK_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1,2,3,4,5,6,7}
|
|
fi
|
|
done
|
|
cd /
|
|
rm -rf $TMP_LOG_PATH
|
|
|
|
}
|
|
|
|
monthly () {
|
|
info "Monthly stat"
|
|
DAY_IN_MONTH=$(cal $(date --date='1 days ago' +"%m %Y") | awk 'NF {DAYS = $NF}; END {print DAYS}')
|
|
info "Copy backup"
|
|
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
|
cd "${LOG_PATH}"
|
|
eval cp -r *access.log.{2..${DAY_IN_MONTH}}.gz "${TMP_LOG_PATH}"
|
|
info "uncompress everything"
|
|
gzip -d ${TMP_LOG_PATH}/*.gz
|
|
mkdir -p ${HTML_MONTH_FILE}
|
|
cd ${TMP_LOG_PATH}
|
|
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
|
do
|
|
info "Generate stat for ${file}"
|
|
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
|
|
eval goaccess --output=${HTML_MONTH_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
|
|
done
|
|
cd /
|
|
rm -rf ${TMP_LOG_PATH}
|
|
|
|
}
|
|
|
|
yearly () {
|
|
info "Yearly stat"
|
|
DAY_IN_YEAR=$(date --date='1 day ago' +"%j")
|
|
info "Copy backup"
|
|
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
|
cd "${LOG_PATH}"
|
|
eval cp -r *access.log.{2..${DAY_IN_YEAR}}.gz "${TMP_LOG_PATH}"
|
|
info "uncompress everything"
|
|
gzip -d ${TMP_LOG_PATH}/*.gz
|
|
mkdir -p ${HTML_YEAR_FILE}
|
|
cd ${TMP_LOG_PATH}
|
|
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
|
do
|
|
info "Generate stat for ${file}"
|
|
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
|
|
eval goaccess --output=${HTML_YEAR_FILE}/${file}.html --ignore-panel=HOSTS --exclude-ip 10.0.0.0/8 --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
|
|
done
|
|
cd /
|
|
rm -rf ${TMP_LOG_PATH}
|
|
}
|
|
|
|
case "${1}" in
|
|
daily)
|
|
daily
|
|
;;
|
|
weekly)
|
|
weekly
|
|
;;
|
|
monthly)
|
|
monthly
|
|
;;
|
|
yearly)
|
|
yearly
|
|
;;
|
|
*)
|
|
usage
|
|
|
|
esac
|
|
info "umount backup"
|
|
/usr/bin/borg umount ${MOUNTPATH}
|
|
|