Add create_stats.sh
This commit is contained in:
parent
48bf26603a
commit
cce33a471a
|
@ -0,0 +1,172 @@
|
|||
#!/bin/bash
|
||||
## author : Dryusdan
|
||||
## date : 22/10/2019
|
||||
## description : A goaccess stat generator
|
||||
## usage : ./create_stats.sh {daily|weekly|monthly|yearly}
|
||||
|
||||
## Bash strict mode ####################################
|
||||
set -o errexit # abort on nonzero exitstatus
|
||||
set -o nounset # abort on unbound variable
|
||||
set -o pipefail # don't hide errors within pipes
|
||||
|
||||
## Bash color ##########################################
|
||||
# Set colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[00;32m'
|
||||
YELLOW='\033[00;33m'
|
||||
BLUE='\033[00;34m'
|
||||
PURPLE='\033[00;35m'
|
||||
CYAN='\033[00;36m'
|
||||
LIGHTGRAY='\033[00;37m'
|
||||
LRED='\033[01;31m'
|
||||
LGREEN='\033[01;32m'
|
||||
LYELLOW='\033[01;33m'
|
||||
LBLUE='\033[01;34m'
|
||||
LPURPLE='\033[01;35m'
|
||||
LCYAN='\033[01;36m'
|
||||
WHITE='\033[01;37m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
## Logs ################################################
|
||||
readonly SCRIPTNAME="$(basename "$0")"
|
||||
info() { echo -e "${LBLUE}[INFO] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
||||
warning() { echo -e "${YELLOW}[WARNING] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
||||
error() { echo -e "${LRED}[ERROR] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; }
|
||||
fatal() { echo -e "${RED}[FATAL] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; exit 1 ; }
|
||||
########################################################
|
||||
|
||||
## Define variables ###################################
|
||||
info "Define variables"
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
REPONAME="log_backup"
|
||||
LOG_PATH="/mnt/${REPONAME}-${DATE} 01:00/var/log/nginx"
|
||||
MOUNTPATH="/mnt"
|
||||
TMP_LOG_PATH="/tmp/logs"
|
||||
HTML_FILE="/var/www/stats/www"
|
||||
HTML_DAY_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/daily/%B/%d')"
|
||||
HTML_WEEK_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/weekly/%V')"
|
||||
HTML_MONTH_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/monthly/%B')"
|
||||
HTML_YEAR_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/yearly')"
|
||||
|
||||
info "Create temp folder if not exist"
|
||||
mkdir -p "${TMP_LOG_PATH}"
|
||||
|
||||
if [ -d "${TMP_LOG_PATH}/nginx" ]; then
|
||||
info "Remove ${TMP_LOG_PATH}/archive-nginx"
|
||||
rm -rf "${TMP_LOG_PATH}"
|
||||
mkdir -p "${TMP_LOG_PATH}"
|
||||
fi
|
||||
|
||||
usage () {
|
||||
echo "Usage: ${SCRIPTNAME} {daily|weekly|monthly|yearly}"
|
||||
info "umount backup"
|
||||
/usr/bin/borg umount ${MOUNTPATH}
|
||||
fatal "Bad parameter ${0}"
|
||||
}
|
||||
|
||||
daily (){
|
||||
info "Daily stat"
|
||||
info "Copy backup"
|
||||
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
||||
info "Create final folder"
|
||||
mkdir -p ${HTML_DAY_FILE}
|
||||
cd ${TMP_LOG_PATH}
|
||||
for file in $(ls ${TMP_LOG_PATH})
|
||||
do
|
||||
info "Generate stat for ${file}"
|
||||
domain=$(echo ${file} | sed "s/\.access\.log.*//g")
|
||||
goaccess --output=${HTML_DAY_FILE}/${domain}-$(date --date='1 days ago' '+%Y-%m-%d').html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}
|
||||
done
|
||||
cd /
|
||||
rm -rf ${TMP_LOG_PATH}
|
||||
}
|
||||
|
||||
weekly (){
|
||||
info "Weekly stat"
|
||||
info "Copy backup"
|
||||
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
||||
cp -r "${LOG_PATH}/"*access*.{2,3,4,5,6,7}.gz "${TMP_LOG_PATH}"
|
||||
info "uncompress everything"
|
||||
gzip -d ${TMP_LOG_PATH}/*.gz
|
||||
mkdir -p ${HTML_WEEK_FILE}
|
||||
cd ${TMP_LOG_PATH}
|
||||
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
||||
do
|
||||
info "Generate stat for ${file}"
|
||||
if [ $(ls ${file}".access.log."* | wc -l ) -eq 7 ]; then
|
||||
goaccess --output=${HTML_WEEK_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1,2,3,4,5,6,7}
|
||||
fi
|
||||
done
|
||||
cd /
|
||||
rm -rf $TMP_LOG_PATH
|
||||
|
||||
}
|
||||
|
||||
monthly () {
|
||||
info "Monthly stat"
|
||||
DAY_IN_MONTH=$(cal $(date --date='1 days ago' +"%m %Y") | awk 'NF {DAYS = $NF}; END {print DAYS}')
|
||||
info "Copy backup"
|
||||
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
||||
cd "${LOG_PATH}"
|
||||
eval cp -r *access.log.{2..${DAY_IN_MONTH}}.gz "${TMP_LOG_PATH}"
|
||||
info "uncompress everything"
|
||||
gzip -d ${TMP_LOG_PATH}/*.gz
|
||||
mkdir -p ${HTML_MONTH_FILE}
|
||||
cd ${TMP_LOG_PATH}
|
||||
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
||||
do
|
||||
info "Generate stat for ${file}"
|
||||
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
|
||||
eval goaccess --output=${HTML_MONTH_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
|
||||
done
|
||||
cd /
|
||||
rm -rf ${TMP_LOG_PATH}
|
||||
|
||||
}
|
||||
|
||||
yearly () {
|
||||
info "Yearly stat"
|
||||
DAY_IN_YEAR=$(date --date='1 day ago' +"%j")
|
||||
info "Copy backup"
|
||||
cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}"
|
||||
cd "${LOG_PATH}"
|
||||
eval cp -r *access.log.{2..${DAY_IN_YEAR}}.gz "${TMP_LOG_PATH}"
|
||||
info "uncompress everything"
|
||||
gzip -d ${TMP_LOG_PATH}/*.gz
|
||||
mkdir -p ${HTML_YEAR_FILE}
|
||||
cd ${TMP_LOG_PATH}
|
||||
for file in $(ls | sed "s/\.access\.log.*//g" | uniq)
|
||||
do
|
||||
info "Generate stat for ${file}"
|
||||
NUMBER_FILES=$(ls ${file}".access.log."* | wc -l )
|
||||
eval goaccess --output=${HTML_YEAR_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}}
|
||||
done
|
||||
cd /
|
||||
rm -rf ${TMP_LOG_PATH}
|
||||
}
|
||||
|
||||
|
||||
info "Mount backup"
|
||||
/usr/bin/borg mount /data/borg/${REPONAME}/${REPONAME} ${MOUNTPATH}
|
||||
|
||||
|
||||
|
||||
case "${1}" in
|
||||
daily)
|
||||
daily
|
||||
;;
|
||||
weekly)
|
||||
weekly
|
||||
;;
|
||||
monthly)
|
||||
monthly
|
||||
;;
|
||||
yearly)
|
||||
yearly
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
|
||||
esac
|
||||
info "umount backup"
|
||||
/usr/bin/borg umount ${MOUNTPATH}
|
Loading…
Reference in New Issue