Merge pull request 'Improve read (black usage)' (#4) from develop into main

Reviewed-on: #4
This commit is contained in:
Dryusdan 2022-07-03 16:38:49 +02:00
commit c298cd0ac1
8 changed files with 228 additions and 93 deletions

View File

@ -1,21 +1,18 @@
from settings import Settings
import logging
import logging.config
import sys
from logging import handlers
class Logger:
log = None
def __init__(self, name):
config=Settings()
#logging.config.dictConfig(config.logging)
#self.logger = logging.getLogger(__name__)
self.log = logging.getLogger(name)
self.log.setLevel(logging.DEBUG)
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
format = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(format)
self.log.addHandler(ch)
self.log.debug('Logging configured')
self.log.debug("Logging configured")

View File

@ -7,8 +7,6 @@ from parser import parseBoucles, parseRecord
from settings import Settings
from sql import crud, models, database
import pprint
settings = Settings()
logging = Logger(__name__)
log = logging.log
@ -41,16 +39,19 @@ def populate_boucles():
else:
log.debug("Records is empty, stop it")
break
log.info("Check if new boucles added and move records into comptages table")
log.info(
"Check if new boucles added and move records into comptages table"
)
for id_boucle in crud.get_unaccounted_record_with_exist_boucle():
log.info(f'New boucle detected. ID {id_boucle}')
log.info('Moving record from unaccounted_comptage to comptage')
log.info(f"New boucle detected. ID {id_boucle}")
log.info("Moving record from unaccounted_comptage to comptage")
for record in crud.select_comptage(id_boucle, True):
log.info(f'Moving record with ID {record.id}')
log.info(f"Moving record with ID {record.id}")
crud.create_comptage(record, False)
log.info(f'Delete record with ID {record.id}')
log.info(f"Delete record with ID {record.id}")
crud.delete_comptage(record.id, True)
@app.command()
def prepopulate_today():
today = pendulum.today("UTC")
@ -66,8 +67,12 @@ def prepopulate_today():
data = r.json()
if data["records"] != []:
for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}')
record_date = pendulum.parse(record["fields"]["dateformat"])
log.debug(
f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today:
log.info("Date is good")
parseRecord(record)
@ -79,20 +84,16 @@ def prepopulate_today():
else:
log.debug("Records is empty, stop it")
process = False
pprint.pprint(data["records"])
else:
pprint.pprint(r.status_code)
pprint.pprint(r.text)
log.error(f"API return a bad status code : {r.status_code}")
process = False
@app.command()
def populate_day():
today = pendulum.today("UTC")
# today = pendulum.datetime(2022, 6, 16, tz='UTC')
yesterday = pendulum.yesterday("UTC")
# yesterday = pendulum.datetime(2022, 6, 15, tz='UTC')
good_day = today.subtract(days=2)
limit = 10
@ -106,8 +107,12 @@ def populate_day():
data = r.json()
if data["records"] != []:
for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}')
record_date = pendulum.parse(record["fields"]["dateformat"])
log.debug(
f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today or record_date == yesterday:
log.info("Date is not complete, ignore it")
elif record_date == good_day:
@ -121,21 +126,20 @@ def populate_day():
else:
log.debug("Records is empty, stop it")
process = False
pprint.pprint(data["records"])
else:
pprint.pprint(r.status_code)
pprint.pprint(r.text)
log.error(f"API return a bad status code : {r.status_code}")
process = False
@app.command()
def populate_all(file: str, force: bool = typer.Option(False, help="Force update")):
with open(f"{file}", 'r') as json_file:
def populate_all(
file: str, force: bool = typer.Option(False, help="Force update")
):
with open(f"{file}", "r") as json_file:
data = json.load(json_file)
for record in data:
log.debug(f'record date is {record["fields"]["dateformat"]}')
record_date = pendulum.parse(record["fields"]["dateformat"])
parseRecord(record, force)
@ -157,8 +161,12 @@ def consolidate_week():
data = r.json()
if data["records"] != []:
for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}')
record_date = pendulum.parse(record["fields"]["dateformat"])
log.debug(
f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today or record_date == yesterday:
log.info("Date is not complete, ignore it")
elif good_day_end <= record_date <= good_day_start:
@ -172,10 +180,8 @@ def consolidate_week():
else:
log.debug("Records is empty, stop it")
process = False
pprint.pprint(data["records"])
else:
pprint.pprint(r.status_code)
pprint.pprint(r.text)
log.error(f"API return a bad status code : {r.status_code}")
process = False

View File

@ -1,11 +1,8 @@
import json
import pendulum
import requests
from logger import Logger
from settings import Settings
from sql import crud, models
import pprint
settings = Settings()
logging = Logger(__name__)
@ -20,7 +17,7 @@ def parseBoucles(boucles):
boucleModel = models.Boucle(
id=boucle["fields"]["boucle_num"],
name=boucle["fields"]["libelle"],
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})'
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})',
)
crud.create_boucle(boucleModel)
else:
@ -28,56 +25,95 @@ def parseBoucles(boucles):
boucleModel = models.Boucle(
id=boucle["fields"]["boucle_num"],
name=boucle["fields"]["libelle"],
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})'
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})',
)
crud.update_boucle(boucleModel)
def parseRecord(record, force = False):
def parseRecord(record, force=False):
if not crud.is_boucle_id_exist(record["fields"]["boucle_num"]):
log.warning(f'Boucle {record["fields"]["boucle_num"]} not exist')
log.warning(f'Add this boucle in temp table')
log.warning("Add this boucle in temp table")
unaccounted_table = True
else:
unaccounted_table = False
for hour in ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"]:
for hour in [
"00",
"01",
"02",
"03",
"04",
"05",
"06",
"07",
"08",
"09",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
]:
if hour in record["fields"]:
if record["fields"]["vacances_zone_b"] == "Hors Vacances":
holiday = False
else:
holiday = True
date = pendulum.parse(record["fields"]["dateformat"]).at(int(hour), 0, 0)
if not crud.is_comptage_exist(date, record["fields"]["boucle_num"], unaccounted_table):
date = pendulum.parse(record["fields"]["dateformat"]).at(
int(hour), 0, 0
)
if not crud.is_comptage_exist(
date, record["fields"]["boucle_num"], unaccounted_table
):
comptage = models.Comptage(
id_boucle=record["fields"]["boucle_num"],
datetime=date,
count=record["fields"][hour],
holiday = holiday,
week_day = record["fields"]["jour_de_la_semaine"]
holiday=holiday,
week_day=record["fields"]["jour_de_la_semaine"],
)
crud.create_comptage(comptage, unaccounted_table)
else:
db_comptage = crud.get_comptage_by_date_and_boucle(date, record["fields"]["boucle_num"], unaccounted_table)
log.error(f"Entry already exist ({record['fields']['boucle_num']}, {date})")
log.debug(f'Check if {db_comptage.count} != {record["fields"][hour]}')
db_comptage = crud.get_comptage_by_date_and_boucle(
date, record["fields"]["boucle_num"], unaccounted_table
)
log.error(
f"Entry already exist ({record['fields']['boucle_num']}, {date})"
)
log.debug(
f'Check if {db_comptage.count} != {record["fields"][hour]}'
)
if db_comptage.count != record["fields"][hour] or force:
if force:
log.info(f'Update with force {db_comptage.id}')
log.info(f"Update with force {db_comptage.id}")
else:
log.warning(f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}')
log.warning(
f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}'
)
comptage = models.Comptage(
id=db_comptage.id,
id_boucle=db_comptage.id_boucle,
datetime=db_comptage.datetime,
count=record["fields"][hour],
holiday = holiday,
week_day = record["fields"]["jour_de_la_semaine"]
holiday=holiday,
week_day=record["fields"]["jour_de_la_semaine"],
)
crud.update_comptage(comptage, unaccounted_table)
else:
log.info(f'Skip {db_comptage.id}, already exist')
#else:
log.info(f"Skip {db_comptage.id}, already exist")
# else:
# log.error(f'Boucle {record["fields"]["boucle_num"]} not exist')
else:
log.error(f'{hour} is not present for {record["fields"]["boucle_num"]}')
log.error(
f'{hour} is not present for {record["fields"]["boucle_num"]}'
)

View File

@ -2,13 +2,15 @@ import os
import yaml
from pathlib import Path
class Settings:
db = {}
def __init__(self):
home = str(Path.home())
config_file = os.environ.get('CONFIGPATH', f"{home}/.config/opendata_nm/config.yml")
config_file = os.environ.get(
"CONFIGPATH", f"{home}/.config/opendata_nm/config.yml"
)
with open(f"{config_file}", "r") as stream:
try:
config = yaml.safe_load(stream)

View File

@ -3,83 +3,160 @@ from .database import Database
from sqlalchemy import delete, update, select, text
from sqlalchemy.orm import sessionmaker
import pprint
db=Database()
db = Database()
Session = sessionmaker(db.engine)
def create_boucle(boucle: schemas.Boucle):
with Session() as session:
db_boucle = models.Boucle(id=boucle.id, name=boucle.name, geolocalisation=boucle.geolocalisation)
db_boucle = models.Boucle(
id=boucle.id,
name=boucle.name,
geolocalisation=boucle.geolocalisation,
)
session.add(db_boucle)
session.commit()
return True
return True
def update_boucle(boucle: schemas.Boucle):
with Session() as session:
stmt = update(models.Boucle).where(models.Boucle.id == boucle.id).values(name=boucle.name, geolocalisation=boucle.geolocalisation)
stmt = (
update(models.Boucle)
.where(models.Boucle.id == boucle.id)
.values(name=boucle.name, geolocalisation=boucle.geolocalisation)
)
session.execute(stmt)
return True
def get_unaccounted_record_with_exist_boucle():
with Session() as session:
textual_sql = text("SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;")
textual_sql = text(
"SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;"
)
return session.execute(textual_sql).scalars()
def create_comptage(comptage: schemas.ComptageBase, unaccounted_table):
with Session() as session:
if unaccounted_table:
db_comptage = models.unaccounted_Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday)
db_comptage = models.unaccounted_Comptage(
id_boucle=comptage.id_boucle,
datetime=comptage.datetime,
count=comptage.count,
week_day=comptage.week_day,
holiday=comptage.holiday,
)
else:
db_comptage = models.Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday)
db_comptage = models.Comptage(
id_boucle=comptage.id_boucle,
datetime=comptage.datetime,
count=comptage.count,
week_day=comptage.week_day,
holiday=comptage.holiday,
)
session.add(db_comptage)
session.commit()
return True
return True
def is_boucle_id_exist(id):
with Session() as session:
return session.query(models.Boucle.id).filter_by(id=id).first() is not None
return (
session.query(models.Boucle.id).filter_by(id=id).first()
is not None
)
def is_comptage_exist(date, id_boucle, unaccounted_table):
with Session() as session:
if unaccounted_table:
return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None
return (
session.query(models.unaccounted_Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
is not None
)
else:
return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None
return (
session.query(models.Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
is not None
)
def get_comptage_by_date_and_boucle(date, id_boucle, unaccounted_table):
with Session() as session:
if unaccounted_table:
return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first()
return (
session.query(models.unaccounted_Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
)
else:
return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first()
return (
session.query(models.Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
)
def update_comptage(comptage: schemas.ComptageBase, unaccounted_table):
with Session() as session:
if unaccounted_table:
stmt = update(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count)
stmt = (
update(models.unaccounted_Comptage)
.where(models.unaccounted_Comptage.id == comptage.id)
.values(
holiday=comptage.holiday,
week_day=comptage.week_day,
count=comptage.count,
)
)
else:
stmt = update(models.Comptage).where(models.Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count)
stmt = (
update(models.Comptage)
.where(models.Comptage.id == comptage.id)
.values(
holiday=comptage.holiday,
week_day=comptage.week_day,
count=comptage.count,
)
)
session.execute(stmt)
session.commit()
return True
def delete_comptage(comptage_id, unaccounted_table):
with Session() as session:
if unaccounted_table:
stmt = delete(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage_id)
stmt = delete(models.unaccounted_Comptage).where(
models.unaccounted_Comptage.id == comptage_id
)
else:
stmt = delete(models.Comptage).where(models.Comptage.id == comptage_id)
stmt = delete(models.Comptage).where(
models.Comptage.id == comptage_id
)
session.execute(stmt)
session.commit()
return True
def select_comptage(id_boucle, unaccounted_table):
with Session() as session:
if unaccounted_table:
stmt = select(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle)
stmt = select(models.unaccounted_Comptage).where(
models.unaccounted_Comptage.id_boucle == id_boucle
)
else:
stmt = select(models.Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle)
return session.execute(stmt, execution_options={"prebuffer_rows": True}).scalars().all()
stmt = select(models.Comptage).where(
models.unaccounted_Comptage.id_boucle == id_boucle
)
return (
session.execute(stmt, execution_options={"prebuffer_rows": True})
.scalars()
.all()
)

View File

@ -2,11 +2,14 @@ from sqlalchemy import create_engine
from logger import Logger
from settings import Settings
settings=Settings()
settings = Settings()
db = settings.db
class Database:
engine = None
def __init__(self):
self.engine = create_engine(f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}')
self.engine = create_engine(
f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}'
)

View File

@ -1,33 +1,43 @@
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, func, Boolean
from sqlalchemy import (
Column,
Integer,
String,
DateTime,
ForeignKey,
func,
Boolean,
)
from sqlalchemy.orm import declarative_base
from geoalchemy2 import Geometry
Base = declarative_base()
class Boucle(Base):
__tablename__ = 'boucles'
__tablename__ = "boucles"
id = Column(Integer, primary_key=True)
name = Column(String)
geolocalisation = Column(Geometry('POINT'))
geolocalisation = Column(Geometry("POINT"))
class Comptage(Base):
__tablename__ = 'comptages'
__tablename__ = "comptages"
id = Column(Integer, primary_key=True)
id_boucle = Column(Integer, ForeignKey("boucles.id"))
datetime = Column('date', DateTime(timezone=True), default=func.now())
datetime = Column("date", DateTime(timezone=True), default=func.now())
week_day = Column(Integer)
holiday = Column(Boolean)
count = Column(Integer)
class unaccounted_Comptage(Base):
__tablename__ = 'unaccounted_comptage'
__tablename__ = "unaccounted_comptage"
id = Column(Integer, primary_key=True)
id_boucle = Column(Integer)
datetime = Column('date', DateTime(timezone=True), default=func.now())
datetime = Column("date", DateTime(timezone=True), default=func.now())
week_day = Column(Integer)
holiday = Column(Boolean)
count = Column(Integer)

View File

@ -1,20 +1,24 @@
from datetime import datetime
from pydantic import BaseModel
class Boucle(BaseModel):
id: int
name: str
geolocalisation: str
class ComptageBase(BaseModel):
id_boucle: int
datetime: datetime
count: int
week_day = int
holiday = bool
holiday = bool
class ComptageCreate(ComptageBase):
pass
class Comptage(ComptageBase):
id: int