diff --git a/src/logger.py b/src/logger.py index a1dbad6..3ed02aa 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,21 +1,18 @@ -from settings import Settings import logging import logging.config import sys -from logging import handlers class Logger: log = None def __init__(self, name): - config=Settings() - #logging.config.dictConfig(config.logging) - #self.logger = logging.getLogger(__name__) self.log = logging.getLogger(name) self.log.setLevel(logging.DEBUG) - format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + format = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) ch = logging.StreamHandler(sys.stdout) ch.setFormatter(format) self.log.addHandler(ch) - self.log.debug('Logging configured') + self.log.debug("Logging configured") diff --git a/src/main.py b/src/main.py index 3f33b82..a65a791 100644 --- a/src/main.py +++ b/src/main.py @@ -7,8 +7,6 @@ from parser import parseBoucles, parseRecord from settings import Settings from sql import crud, models, database -import pprint - settings = Settings() logging = Logger(__name__) log = logging.log @@ -41,16 +39,19 @@ def populate_boucles(): else: log.debug("Records is empty, stop it") break - log.info("Check if new boucles added and move records into comptages table") + log.info( + "Check if new boucles added and move records into comptages table" + ) for id_boucle in crud.get_unaccounted_record_with_exist_boucle(): - log.info(f'New boucle detected. ID {id_boucle}') - log.info('Moving record from unaccounted_comptage to comptage') + log.info(f"New boucle detected. ID {id_boucle}") + log.info("Moving record from unaccounted_comptage to comptage") for record in crud.select_comptage(id_boucle, True): - log.info(f'Moving record with ID {record.id}') + log.info(f"Moving record with ID {record.id}") crud.create_comptage(record, False) - log.info(f'Delete record with ID {record.id}') + log.info(f"Delete record with ID {record.id}") crud.delete_comptage(record.id, True) + @app.command() def prepopulate_today(): today = pendulum.today("UTC") @@ -66,8 +67,12 @@ def prepopulate_today(): data = r.json() if data["records"] != []: for record in data["records"]: - log.debug(f'record date is {record["fields"]["dateformat"]}') - record_date = pendulum.parse(record["fields"]["dateformat"]) + log.debug( + f'record date is {record["fields"]["dateformat"]}' + ) + record_date = pendulum.parse( + record["fields"]["dateformat"] + ) if record_date == today: log.info("Date is good") parseRecord(record) @@ -79,20 +84,16 @@ def prepopulate_today(): else: log.debug("Records is empty, stop it") process = False - pprint.pprint(data["records"]) else: - pprint.pprint(r.status_code) - pprint.pprint(r.text) + log.error(f"API return a bad status code : {r.status_code}") process = False @app.command() def populate_day(): today = pendulum.today("UTC") - # today = pendulum.datetime(2022, 6, 16, tz='UTC') yesterday = pendulum.yesterday("UTC") - # yesterday = pendulum.datetime(2022, 6, 15, tz='UTC') good_day = today.subtract(days=2) limit = 10 @@ -106,8 +107,12 @@ def populate_day(): data = r.json() if data["records"] != []: for record in data["records"]: - log.debug(f'record date is {record["fields"]["dateformat"]}') - record_date = pendulum.parse(record["fields"]["dateformat"]) + log.debug( + f'record date is {record["fields"]["dateformat"]}' + ) + record_date = pendulum.parse( + record["fields"]["dateformat"] + ) if record_date == today or record_date == yesterday: log.info("Date is not complete, ignore it") elif record_date == good_day: @@ -121,21 +126,20 @@ def populate_day(): else: log.debug("Records is empty, stop it") process = False - pprint.pprint(data["records"]) else: - pprint.pprint(r.status_code) - pprint.pprint(r.text) + log.error(f"API return a bad status code : {r.status_code}") process = False @app.command() -def populate_all(file: str, force: bool = typer.Option(False, help="Force update")): - with open(f"{file}", 'r') as json_file: +def populate_all( + file: str, force: bool = typer.Option(False, help="Force update") +): + with open(f"{file}", "r") as json_file: data = json.load(json_file) for record in data: log.debug(f'record date is {record["fields"]["dateformat"]}') - record_date = pendulum.parse(record["fields"]["dateformat"]) parseRecord(record, force) @@ -157,8 +161,12 @@ def consolidate_week(): data = r.json() if data["records"] != []: for record in data["records"]: - log.debug(f'record date is {record["fields"]["dateformat"]}') - record_date = pendulum.parse(record["fields"]["dateformat"]) + log.debug( + f'record date is {record["fields"]["dateformat"]}' + ) + record_date = pendulum.parse( + record["fields"]["dateformat"] + ) if record_date == today or record_date == yesterday: log.info("Date is not complete, ignore it") elif good_day_end <= record_date <= good_day_start: @@ -172,10 +180,8 @@ def consolidate_week(): else: log.debug("Records is empty, stop it") process = False - pprint.pprint(data["records"]) else: - pprint.pprint(r.status_code) - pprint.pprint(r.text) + log.error(f"API return a bad status code : {r.status_code}") process = False diff --git a/src/parser.py b/src/parser.py index 12d8736..b79bc42 100644 --- a/src/parser.py +++ b/src/parser.py @@ -1,11 +1,8 @@ -import json import pendulum -import requests from logger import Logger from settings import Settings from sql import crud, models -import pprint settings = Settings() logging = Logger(__name__) @@ -20,7 +17,7 @@ def parseBoucles(boucles): boucleModel = models.Boucle( id=boucle["fields"]["boucle_num"], name=boucle["fields"]["libelle"], - geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})' + geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})', ) crud.create_boucle(boucleModel) else: @@ -28,56 +25,95 @@ def parseBoucles(boucles): boucleModel = models.Boucle( id=boucle["fields"]["boucle_num"], name=boucle["fields"]["libelle"], - geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})' + geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})', ) crud.update_boucle(boucleModel) -def parseRecord(record, force = False): + +def parseRecord(record, force=False): if not crud.is_boucle_id_exist(record["fields"]["boucle_num"]): log.warning(f'Boucle {record["fields"]["boucle_num"]} not exist') - log.warning(f'Add this boucle in temp table') + log.warning("Add this boucle in temp table") unaccounted_table = True else: unaccounted_table = False - for hour in ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"]: + for hour in [ + "00", + "01", + "02", + "03", + "04", + "05", + "06", + "07", + "08", + "09", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17", + "18", + "19", + "20", + "21", + "22", + "23", + ]: if hour in record["fields"]: if record["fields"]["vacances_zone_b"] == "Hors Vacances": holiday = False else: holiday = True - date = pendulum.parse(record["fields"]["dateformat"]).at(int(hour), 0, 0) - if not crud.is_comptage_exist(date, record["fields"]["boucle_num"], unaccounted_table): + date = pendulum.parse(record["fields"]["dateformat"]).at( + int(hour), 0, 0 + ) + if not crud.is_comptage_exist( + date, record["fields"]["boucle_num"], unaccounted_table + ): comptage = models.Comptage( id_boucle=record["fields"]["boucle_num"], datetime=date, count=record["fields"][hour], - holiday = holiday, - week_day = record["fields"]["jour_de_la_semaine"] + holiday=holiday, + week_day=record["fields"]["jour_de_la_semaine"], ) crud.create_comptage(comptage, unaccounted_table) else: - db_comptage = crud.get_comptage_by_date_and_boucle(date, record["fields"]["boucle_num"], unaccounted_table) - log.error(f"Entry already exist ({record['fields']['boucle_num']}, {date})") - log.debug(f'Check if {db_comptage.count} != {record["fields"][hour]}') + db_comptage = crud.get_comptage_by_date_and_boucle( + date, record["fields"]["boucle_num"], unaccounted_table + ) + log.error( + f"Entry already exist ({record['fields']['boucle_num']}, {date})" + ) + log.debug( + f'Check if {db_comptage.count} != {record["fields"][hour]}' + ) if db_comptage.count != record["fields"][hour] or force: if force: - log.info(f'Update with force {db_comptage.id}') + log.info(f"Update with force {db_comptage.id}") else: - log.warning(f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}') + log.warning( + f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}' + ) comptage = models.Comptage( id=db_comptage.id, id_boucle=db_comptage.id_boucle, datetime=db_comptage.datetime, count=record["fields"][hour], - holiday = holiday, - week_day = record["fields"]["jour_de_la_semaine"] + holiday=holiday, + week_day=record["fields"]["jour_de_la_semaine"], ) crud.update_comptage(comptage, unaccounted_table) else: - log.info(f'Skip {db_comptage.id}, already exist') - #else: + log.info(f"Skip {db_comptage.id}, already exist") + # else: # log.error(f'Boucle {record["fields"]["boucle_num"]} not exist') else: - log.error(f'{hour} is not present for {record["fields"]["boucle_num"]}') - + log.error( + f'{hour} is not present for {record["fields"]["boucle_num"]}' + ) diff --git a/src/settings.py b/src/settings.py index 390bf8e..ce7bceb 100644 --- a/src/settings.py +++ b/src/settings.py @@ -2,13 +2,15 @@ import os import yaml from pathlib import Path + class Settings: db = {} - def __init__(self): home = str(Path.home()) - config_file = os.environ.get('CONFIGPATH', f"{home}/.config/opendata_nm/config.yml") + config_file = os.environ.get( + "CONFIGPATH", f"{home}/.config/opendata_nm/config.yml" + ) with open(f"{config_file}", "r") as stream: try: config = yaml.safe_load(stream) diff --git a/src/sql/crud.py b/src/sql/crud.py index 0dcce31..75dc19f 100644 --- a/src/sql/crud.py +++ b/src/sql/crud.py @@ -3,83 +3,160 @@ from .database import Database from sqlalchemy import delete, update, select, text from sqlalchemy.orm import sessionmaker -import pprint - -db=Database() +db = Database() Session = sessionmaker(db.engine) + def create_boucle(boucle: schemas.Boucle): with Session() as session: - db_boucle = models.Boucle(id=boucle.id, name=boucle.name, geolocalisation=boucle.geolocalisation) + db_boucle = models.Boucle( + id=boucle.id, + name=boucle.name, + geolocalisation=boucle.geolocalisation, + ) session.add(db_boucle) session.commit() - return True + return True + def update_boucle(boucle: schemas.Boucle): with Session() as session: - stmt = update(models.Boucle).where(models.Boucle.id == boucle.id).values(name=boucle.name, geolocalisation=boucle.geolocalisation) + stmt = ( + update(models.Boucle) + .where(models.Boucle.id == boucle.id) + .values(name=boucle.name, geolocalisation=boucle.geolocalisation) + ) session.execute(stmt) return True + def get_unaccounted_record_with_exist_boucle(): with Session() as session: - textual_sql = text("SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;") + textual_sql = text( + "SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;" + ) return session.execute(textual_sql).scalars() def create_comptage(comptage: schemas.ComptageBase, unaccounted_table): with Session() as session: if unaccounted_table: - db_comptage = models.unaccounted_Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday) + db_comptage = models.unaccounted_Comptage( + id_boucle=comptage.id_boucle, + datetime=comptage.datetime, + count=comptage.count, + week_day=comptage.week_day, + holiday=comptage.holiday, + ) else: - db_comptage = models.Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday) + db_comptage = models.Comptage( + id_boucle=comptage.id_boucle, + datetime=comptage.datetime, + count=comptage.count, + week_day=comptage.week_day, + holiday=comptage.holiday, + ) session.add(db_comptage) session.commit() - return True + return True + def is_boucle_id_exist(id): with Session() as session: - return session.query(models.Boucle.id).filter_by(id=id).first() is not None + return ( + session.query(models.Boucle.id).filter_by(id=id).first() + is not None + ) + def is_comptage_exist(date, id_boucle, unaccounted_table): with Session() as session: if unaccounted_table: - return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None + return ( + session.query(models.unaccounted_Comptage) + .filter_by(datetime=date, id_boucle=id_boucle) + .first() + is not None + ) else: - return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None + return ( + session.query(models.Comptage) + .filter_by(datetime=date, id_boucle=id_boucle) + .first() + is not None + ) + def get_comptage_by_date_and_boucle(date, id_boucle, unaccounted_table): with Session() as session: if unaccounted_table: - return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() + return ( + session.query(models.unaccounted_Comptage) + .filter_by(datetime=date, id_boucle=id_boucle) + .first() + ) else: - return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() + return ( + session.query(models.Comptage) + .filter_by(datetime=date, id_boucle=id_boucle) + .first() + ) + def update_comptage(comptage: schemas.ComptageBase, unaccounted_table): with Session() as session: if unaccounted_table: - stmt = update(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count) + stmt = ( + update(models.unaccounted_Comptage) + .where(models.unaccounted_Comptage.id == comptage.id) + .values( + holiday=comptage.holiday, + week_day=comptage.week_day, + count=comptage.count, + ) + ) else: - stmt = update(models.Comptage).where(models.Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count) + stmt = ( + update(models.Comptage) + .where(models.Comptage.id == comptage.id) + .values( + holiday=comptage.holiday, + week_day=comptage.week_day, + count=comptage.count, + ) + ) session.execute(stmt) session.commit() return True + def delete_comptage(comptage_id, unaccounted_table): with Session() as session: if unaccounted_table: - stmt = delete(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage_id) + stmt = delete(models.unaccounted_Comptage).where( + models.unaccounted_Comptage.id == comptage_id + ) else: - stmt = delete(models.Comptage).where(models.Comptage.id == comptage_id) + stmt = delete(models.Comptage).where( + models.Comptage.id == comptage_id + ) session.execute(stmt) session.commit() return True + def select_comptage(id_boucle, unaccounted_table): with Session() as session: if unaccounted_table: - stmt = select(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle) + stmt = select(models.unaccounted_Comptage).where( + models.unaccounted_Comptage.id_boucle == id_boucle + ) else: - stmt = select(models.Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle) - return session.execute(stmt, execution_options={"prebuffer_rows": True}).scalars().all() - + stmt = select(models.Comptage).where( + models.unaccounted_Comptage.id_boucle == id_boucle + ) + return ( + session.execute(stmt, execution_options={"prebuffer_rows": True}) + .scalars() + .all() + ) diff --git a/src/sql/database.py b/src/sql/database.py index 608ff4e..a11d051 100644 --- a/src/sql/database.py +++ b/src/sql/database.py @@ -2,11 +2,14 @@ from sqlalchemy import create_engine from logger import Logger from settings import Settings -settings=Settings() +settings = Settings() db = settings.db + class Database: engine = None def __init__(self): - self.engine = create_engine(f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}') + self.engine = create_engine( + f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}' + ) diff --git a/src/sql/models.py b/src/sql/models.py index 5cc9308..370c36e 100644 --- a/src/sql/models.py +++ b/src/sql/models.py @@ -1,33 +1,43 @@ -from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, func, Boolean +from sqlalchemy import ( + Column, + Integer, + String, + DateTime, + ForeignKey, + func, + Boolean, +) from sqlalchemy.orm import declarative_base from geoalchemy2 import Geometry Base = declarative_base() + class Boucle(Base): - __tablename__ = 'boucles' + __tablename__ = "boucles" id = Column(Integer, primary_key=True) name = Column(String) - geolocalisation = Column(Geometry('POINT')) + geolocalisation = Column(Geometry("POINT")) + class Comptage(Base): - __tablename__ = 'comptages' + __tablename__ = "comptages" id = Column(Integer, primary_key=True) id_boucle = Column(Integer, ForeignKey("boucles.id")) - datetime = Column('date', DateTime(timezone=True), default=func.now()) + datetime = Column("date", DateTime(timezone=True), default=func.now()) week_day = Column(Integer) holiday = Column(Boolean) count = Column(Integer) + class unaccounted_Comptage(Base): - __tablename__ = 'unaccounted_comptage' + __tablename__ = "unaccounted_comptage" id = Column(Integer, primary_key=True) id_boucle = Column(Integer) - datetime = Column('date', DateTime(timezone=True), default=func.now()) + datetime = Column("date", DateTime(timezone=True), default=func.now()) week_day = Column(Integer) holiday = Column(Boolean) count = Column(Integer) - diff --git a/src/sql/schemas.py b/src/sql/schemas.py index b02876a..ca8b085 100644 --- a/src/sql/schemas.py +++ b/src/sql/schemas.py @@ -1,20 +1,24 @@ from datetime import datetime from pydantic import BaseModel + class Boucle(BaseModel): id: int name: str geolocalisation: str + class ComptageBase(BaseModel): id_boucle: int datetime: datetime count: int week_day = int - holiday = bool + holiday = bool + class ComptageCreate(ComptageBase): pass + class Comptage(ComptageBase): id: int