diff --git a/create_stats.sh b/create_stats.sh new file mode 100644 index 0000000..12fd093 --- /dev/null +++ b/create_stats.sh @@ -0,0 +1,172 @@ +#!/bin/bash +## author : Dryusdan +## date : 22/10/2019 +## description : A goaccess stat generator +## usage : ./create_stats.sh {daily|weekly|monthly|yearly} + +## Bash strict mode #################################### +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +## Bash color ########################################## +# Set colors +RED='\033[0;31m' +GREEN='\033[00;32m' +YELLOW='\033[00;33m' +BLUE='\033[00;34m' +PURPLE='\033[00;35m' +CYAN='\033[00;36m' +LIGHTGRAY='\033[00;37m' +LRED='\033[01;31m' +LGREEN='\033[01;32m' +LYELLOW='\033[01;33m' +LBLUE='\033[01;34m' +LPURPLE='\033[01;35m' +LCYAN='\033[01;36m' +WHITE='\033[01;37m' +NC='\033[0m' # No Color + +## Logs ################################################ +readonly SCRIPTNAME="$(basename "$0")" +info() { echo -e "${LBLUE}[INFO] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; } +warning() { echo -e "${YELLOW}[WARNING] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; } +error() { echo -e "${LRED}[ERROR] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; } +fatal() { echo -e "${RED}[FATAL] $* ${NC}" | logger --tag "${SCRIPTNAME}" --stderr ; exit 1 ; } +######################################################## + +## Define variables ################################### +info "Define variables" +DATE=$(date '+%Y-%m-%d') +REPONAME="log_backup" +LOG_PATH="/mnt/${REPONAME}-${DATE} 01:00/var/log/nginx" +MOUNTPATH="/mnt" +TMP_LOG_PATH="/tmp/logs" +HTML_FILE="/var/www/stats/www" +HTML_DAY_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/daily/%B/%d')" +HTML_WEEK_FILE="$HTML_FILE/$(date --date='1 day ago' '+%Y/weekly/%V')" +HTML_MONTH_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/monthly/%B')" +HTML_YEAR_FILE="${HTML_FILE}/$(date --date='1 day ago' '+%Y/yearly')" + +info "Create temp folder if not exist" +mkdir -p "${TMP_LOG_PATH}" + +if [ -d "${TMP_LOG_PATH}/nginx" ]; then + info "Remove ${TMP_LOG_PATH}/archive-nginx" + rm -rf "${TMP_LOG_PATH}" + mkdir -p "${TMP_LOG_PATH}" +fi + +usage () { + echo "Usage: ${SCRIPTNAME} {daily|weekly|monthly|yearly}" + info "umount backup" + /usr/bin/borg umount ${MOUNTPATH} + fatal "Bad parameter ${0}" +} + +daily (){ + info "Daily stat" + info "Copy backup" + cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}" + info "Create final folder" + mkdir -p ${HTML_DAY_FILE} + cd ${TMP_LOG_PATH} + for file in $(ls ${TMP_LOG_PATH}) + do + info "Generate stat for ${file}" + domain=$(echo ${file} | sed "s/\.access\.log.*//g") + goaccess --output=${HTML_DAY_FILE}/${domain}-$(date --date='1 days ago' '+%Y-%m-%d').html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file} + done + cd / + rm -rf ${TMP_LOG_PATH} +} + +weekly (){ + info "Weekly stat" + info "Copy backup" + cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}" + cp -r "${LOG_PATH}/"*access*.{2,3,4,5,6,7}.gz "${TMP_LOG_PATH}" + info "uncompress everything" + gzip -d ${TMP_LOG_PATH}/*.gz + mkdir -p ${HTML_WEEK_FILE} + cd ${TMP_LOG_PATH} + for file in $(ls | sed "s/\.access\.log.*//g" | uniq) + do + info "Generate stat for ${file}" + if [ $(ls ${file}".access.log."* | wc -l ) -eq 7 ]; then + goaccess --output=${HTML_WEEK_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1,2,3,4,5,6,7} + fi + done + cd / + rm -rf $TMP_LOG_PATH + +} + +monthly () { + info "Monthly stat" + DAY_IN_MONTH=$(cal $(date --date='1 days ago' +"%m %Y") | awk 'NF {DAYS = $NF}; END {print DAYS}') + info "Copy backup" + cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}" + cd "${LOG_PATH}" + eval cp -r *access.log.{2..${DAY_IN_MONTH}}.gz "${TMP_LOG_PATH}" + info "uncompress everything" + gzip -d ${TMP_LOG_PATH}/*.gz + mkdir -p ${HTML_MONTH_FILE} + cd ${TMP_LOG_PATH} + for file in $(ls | sed "s/\.access\.log.*//g" | uniq) + do + info "Generate stat for ${file}" + NUMBER_FILES=$(ls ${file}".access.log."* | wc -l ) + eval goaccess --output=${HTML_MONTH_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}} + done + cd / + rm -rf ${TMP_LOG_PATH} + +} + +yearly () { + info "Yearly stat" + DAY_IN_YEAR=$(date --date='1 day ago' +"%j") + info "Copy backup" + cp -r "${LOG_PATH}/"*access*.1 "${TMP_LOG_PATH}" + cd "${LOG_PATH}" + eval cp -r *access.log.{2..${DAY_IN_YEAR}}.gz "${TMP_LOG_PATH}" + info "uncompress everything" + gzip -d ${TMP_LOG_PATH}/*.gz + mkdir -p ${HTML_YEAR_FILE} + cd ${TMP_LOG_PATH} + for file in $(ls | sed "s/\.access\.log.*//g" | uniq) + do + info "Generate stat for ${file}" + NUMBER_FILES=$(ls ${file}".access.log."* | wc -l ) + eval goaccess --output=${HTML_YEAR_FILE}/${file}.html --std-geoip --anonymize-ip --ignore-crawlers --log-format=COMBINED ${file}".access.log."{1..${NUMBER_FILES}} + done + cd / + rm -rf ${TMP_LOG_PATH} +} + + +info "Mount backup" +/usr/bin/borg mount /data/borg/${REPONAME}/${REPONAME} ${MOUNTPATH} + + + +case "${1}" in + daily) + daily + ;; + weekly) + weekly + ;; + monthly) + monthly + ;; + yearly) + yearly + ;; + *) + usage + +esac +info "umount backup" +/usr/bin/borg umount ${MOUNTPATH}