Dryusdan
/
opendata-nm-exporter
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve read (black usage)

This commit is contained in:
Dryusdan 2022-07-03 16:38:05 +02:00
parent c7b9f856cc
commit 8123e62a64
8 changed files with 228 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/logger.py
View File

@ -1,21 +1,18 @@
from settings import Settings
import logging import logging
import logging.config import logging.config
import sys import sys
from logging import handlers
class Logger: class Logger:
log = None log = None
def __init__(self, name): def __init__(self, name):
config=Settings()
#logging.config.dictConfig(config.logging)
#self.logger = logging.getLogger(__name__)
self.log = logging.getLogger(name) self.log = logging.getLogger(name)
self.log.setLevel(logging.DEBUG) self.log.setLevel(logging.DEBUG)
format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") format = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
ch = logging.StreamHandler(sys.stdout) ch = logging.StreamHandler(sys.stdout)
ch.setFormatter(format) ch.setFormatter(format)
self.log.addHandler(ch) self.log.addHandler(ch)
self.log.debug('Logging configured') self.log.debug("Logging configured")

60
src/main.py
View File

@ -7,8 +7,6 @@ from parser import parseBoucles, parseRecord
from settings import Settings from settings import Settings
from sql import crud, models, database from sql import crud, models, database
import pprint
settings = Settings() settings = Settings()
logging = Logger(__name__) logging = Logger(__name__)
log = logging.log log = logging.log
@ -41,16 +39,19 @@ def populate_boucles():
else: else:
log.debug("Records is empty, stop it") log.debug("Records is empty, stop it")
break break
log.info("Check if new boucles added and move records into comptages table") log.info(
"Check if new boucles added and move records into comptages table"
)
for id_boucle in crud.get_unaccounted_record_with_exist_boucle(): for id_boucle in crud.get_unaccounted_record_with_exist_boucle():
log.info(f'New boucle detected. ID {id_boucle}') log.info(f"New boucle detected. ID {id_boucle}")
log.info('Moving record from unaccounted_comptage to comptage') log.info("Moving record from unaccounted_comptage to comptage")
for record in crud.select_comptage(id_boucle, True): for record in crud.select_comptage(id_boucle, True):
log.info(f'Moving record with ID {record.id}') log.info(f"Moving record with ID {record.id}")
crud.create_comptage(record, False) crud.create_comptage(record, False)
log.info(f'Delete record with ID {record.id}') log.info(f"Delete record with ID {record.id}")
crud.delete_comptage(record.id, True) crud.delete_comptage(record.id, True)
@app.command() @app.command()
def prepopulate_today(): def prepopulate_today():
today = pendulum.today("UTC") today = pendulum.today("UTC")
@ -66,8 +67,12 @@ def prepopulate_today():
data = r.json() data = r.json()
if data["records"] != []: if data["records"] != []:
for record in data["records"]: for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}') log.debug(
record_date = pendulum.parse(record["fields"]["dateformat"]) f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today: if record_date == today:
log.info("Date is good") log.info("Date is good")
parseRecord(record) parseRecord(record)
@ -79,20 +84,16 @@ def prepopulate_today():
else: else:
log.debug("Records is empty, stop it") log.debug("Records is empty, stop it")
process = False process = False
pprint.pprint(data["records"])
else: else:
pprint.pprint(r.status_code) log.error(f"API return a bad status code : {r.status_code}")
pprint.pprint(r.text)
process = False process = False
@app.command() @app.command()
def populate_day(): def populate_day():
today = pendulum.today("UTC") today = pendulum.today("UTC")
# today = pendulum.datetime(2022, 6, 16, tz='UTC')
yesterday = pendulum.yesterday("UTC") yesterday = pendulum.yesterday("UTC")
# yesterday = pendulum.datetime(2022, 6, 15, tz='UTC')
good_day = today.subtract(days=2) good_day = today.subtract(days=2)
limit = 10 limit = 10
@ -106,8 +107,12 @@ def populate_day():
data = r.json() data = r.json()
if data["records"] != []: if data["records"] != []:
for record in data["records"]: for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}') log.debug(
record_date = pendulum.parse(record["fields"]["dateformat"]) f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today or record_date == yesterday: if record_date == today or record_date == yesterday:
log.info("Date is not complete, ignore it") log.info("Date is not complete, ignore it")
elif record_date == good_day: elif record_date == good_day:
@ -121,21 +126,20 @@ def populate_day():
else: else:
log.debug("Records is empty, stop it") log.debug("Records is empty, stop it")
process = False process = False
pprint.pprint(data["records"])
else: else:
pprint.pprint(r.status_code) log.error(f"API return a bad status code : {r.status_code}")
pprint.pprint(r.text)
process = False process = False
@app.command() @app.command()
def populate_all(file: str, force: bool = typer.Option(False, help="Force update")): def populate_all(
with open(f"{file}", 'r') as json_file: file: str, force: bool = typer.Option(False, help="Force update")
):
with open(f"{file}", "r") as json_file:
data = json.load(json_file) data = json.load(json_file)
for record in data: for record in data:
log.debug(f'record date is {record["fields"]["dateformat"]}') log.debug(f'record date is {record["fields"]["dateformat"]}')
record_date = pendulum.parse(record["fields"]["dateformat"])
parseRecord(record, force) parseRecord(record, force)
@ -157,8 +161,12 @@ def consolidate_week():
data = r.json() data = r.json()
if data["records"] != []: if data["records"] != []:
for record in data["records"]: for record in data["records"]:
log.debug(f'record date is {record["fields"]["dateformat"]}') log.debug(
record_date = pendulum.parse(record["fields"]["dateformat"]) f'record date is {record["fields"]["dateformat"]}'
)
record_date = pendulum.parse(
record["fields"]["dateformat"]
)
if record_date == today or record_date == yesterday: if record_date == today or record_date == yesterday:
log.info("Date is not complete, ignore it") log.info("Date is not complete, ignore it")
elif good_day_end <= record_date <= good_day_start: elif good_day_end <= record_date <= good_day_start:
@ -172,10 +180,8 @@ def consolidate_week():
else: else:
log.debug("Records is empty, stop it") log.debug("Records is empty, stop it")
process = False process = False
pprint.pprint(data["records"])
else: else:
pprint.pprint(r.status_code) log.error(f"API return a bad status code : {r.status_code}")
pprint.pprint(r.text)
process = False process = False

82
src/parser.py
View File

@ -1,11 +1,8 @@
import json
import pendulum import pendulum
import requests
from logger import Logger from logger import Logger
from settings import Settings from settings import Settings
from sql import crud, models from sql import crud, models
import pprint
settings = Settings() settings = Settings()
logging = Logger(__name__) logging = Logger(__name__)
@ -20,7 +17,7 @@ def parseBoucles(boucles):
boucleModel = models.Boucle( boucleModel = models.Boucle(
id=boucle["fields"]["boucle_num"], id=boucle["fields"]["boucle_num"],
name=boucle["fields"]["libelle"], name=boucle["fields"]["libelle"],
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})' geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})',
) )
crud.create_boucle(boucleModel) crud.create_boucle(boucleModel)
else: else:
@ -28,56 +25,95 @@ def parseBoucles(boucles):
boucleModel = models.Boucle( boucleModel = models.Boucle(
id=boucle["fields"]["boucle_num"], id=boucle["fields"]["boucle_num"],
name=boucle["fields"]["libelle"], name=boucle["fields"]["libelle"],
geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})' geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})',
) )
crud.update_boucle(boucleModel) crud.update_boucle(boucleModel)
def parseRecord(record, force = False):
def parseRecord(record, force=False):
if not crud.is_boucle_id_exist(record["fields"]["boucle_num"]): if not crud.is_boucle_id_exist(record["fields"]["boucle_num"]):
log.warning(f'Boucle {record["fields"]["boucle_num"]} not exist') log.warning(f'Boucle {record["fields"]["boucle_num"]} not exist')
log.warning(f'Add this boucle in temp table') log.warning("Add this boucle in temp table")
unaccounted_table = True unaccounted_table = True
else: else:
unaccounted_table = False unaccounted_table = False
for hour in ["00", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23"]: for hour in [
"00",
"01",
"02",
"03",
"04",
"05",
"06",
"07",
"08",
"09",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
]:
if hour in record["fields"]: if hour in record["fields"]:
if record["fields"]["vacances_zone_b"] == "Hors Vacances": if record["fields"]["vacances_zone_b"] == "Hors Vacances":
holiday = False holiday = False
else: else:
holiday = True holiday = True
date = pendulum.parse(record["fields"]["dateformat"]).at(int(hour), 0, 0) date = pendulum.parse(record["fields"]["dateformat"]).at(
if not crud.is_comptage_exist(date, record["fields"]["boucle_num"], unaccounted_table): int(hour), 0, 0
)
if not crud.is_comptage_exist(
date, record["fields"]["boucle_num"], unaccounted_table
):
comptage = models.Comptage( comptage = models.Comptage(
id_boucle=record["fields"]["boucle_num"], id_boucle=record["fields"]["boucle_num"],
datetime=date, datetime=date,
count=record["fields"][hour], count=record["fields"][hour],
holiday = holiday, holiday=holiday,
week_day = record["fields"]["jour_de_la_semaine"] week_day=record["fields"]["jour_de_la_semaine"],
) )
crud.create_comptage(comptage, unaccounted_table) crud.create_comptage(comptage, unaccounted_table)
else: else:
db_comptage = crud.get_comptage_by_date_and_boucle(date, record["fields"]["boucle_num"], unaccounted_table) db_comptage = crud.get_comptage_by_date_and_boucle(
log.error(f"Entry already exist ({record['fields']['boucle_num']}, {date})") date, record["fields"]["boucle_num"], unaccounted_table
log.debug(f'Check if {db_comptage.count} != {record["fields"][hour]}') )
log.error(
f"Entry already exist ({record['fields']['boucle_num']}, {date})"
)
log.debug(
f'Check if {db_comptage.count} != {record["fields"][hour]}'
)
if db_comptage.count != record["fields"][hour] or force: if db_comptage.count != record["fields"][hour] or force:
if force: if force:
log.info(f'Update with force {db_comptage.id}') log.info(f"Update with force {db_comptage.id}")
else: else:
log.warning(f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}') log.warning(
f'Entry {db_comptage.id} have a different count (DB : {db_comptage.count}, data {record["field"][hour]}'
)
comptage = models.Comptage( comptage = models.Comptage(
id=db_comptage.id, id=db_comptage.id,
id_boucle=db_comptage.id_boucle, id_boucle=db_comptage.id_boucle,
datetime=db_comptage.datetime, datetime=db_comptage.datetime,
count=record["fields"][hour], count=record["fields"][hour],
holiday = holiday, holiday=holiday,
week_day = record["fields"]["jour_de_la_semaine"] week_day=record["fields"]["jour_de_la_semaine"],
) )
crud.update_comptage(comptage, unaccounted_table) crud.update_comptage(comptage, unaccounted_table)
else: else:
log.info(f'Skip {db_comptage.id}, already exist') log.info(f"Skip {db_comptage.id}, already exist")
#else: # else:
# log.error(f'Boucle {record["fields"]["boucle_num"]} not exist') # log.error(f'Boucle {record["fields"]["boucle_num"]} not exist')
else: else:
log.error(f'{hour} is not present for {record["fields"]["boucle_num"]}') log.error(
f'{hour} is not present for {record["fields"]["boucle_num"]}'
)

6
src/settings.py
View File

@ -2,13 +2,15 @@ import os
import yaml import yaml
from pathlib import Path from pathlib import Path
class Settings: class Settings:
db = {} db = {}
def __init__(self): def __init__(self):
home = str(Path.home()) home = str(Path.home())
config_file = os.environ.get('CONFIGPATH', f"{home}/.config/opendata_nm/config.yml") config_file = os.environ.get(
"CONFIGPATH", f"{home}/.config/opendata_nm/config.yml"
)
with open(f"{config_file}", "r") as stream: with open(f"{config_file}", "r") as stream:
try: try:
config = yaml.safe_load(stream) config = yaml.safe_load(stream)

123
src/sql/crud.py
View File

@ -3,83 +3,160 @@ from .database import Database
from sqlalchemy import delete, update, select, text from sqlalchemy import delete, update, select, text
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
import pprint db = Database()
db=Database()
Session = sessionmaker(db.engine) Session = sessionmaker(db.engine)
def create_boucle(boucle: schemas.Boucle): def create_boucle(boucle: schemas.Boucle):
with Session() as session: with Session() as session:
db_boucle = models.Boucle(id=boucle.id, name=boucle.name, geolocalisation=boucle.geolocalisation) db_boucle = models.Boucle(
id=boucle.id,
name=boucle.name,
geolocalisation=boucle.geolocalisation,
)
session.add(db_boucle) session.add(db_boucle)
session.commit() session.commit()
return True return True
def update_boucle(boucle: schemas.Boucle): def update_boucle(boucle: schemas.Boucle):
with Session() as session: with Session() as session:
stmt = update(models.Boucle).where(models.Boucle.id == boucle.id).values(name=boucle.name, geolocalisation=boucle.geolocalisation) stmt = (
update(models.Boucle)
.where(models.Boucle.id == boucle.id)
.values(name=boucle.name, geolocalisation=boucle.geolocalisation)
)
session.execute(stmt) session.execute(stmt)
return True return True
def get_unaccounted_record_with_exist_boucle(): def get_unaccounted_record_with_exist_boucle():
with Session() as session: with Session() as session:
textual_sql = text("SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;") textual_sql = text(
"SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;"
)
return session.execute(textual_sql).scalars() return session.execute(textual_sql).scalars()
def create_comptage(comptage: schemas.ComptageBase, unaccounted_table): def create_comptage(comptage: schemas.ComptageBase, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
db_comptage = models.unaccounted_Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday) db_comptage = models.unaccounted_Comptage(
id_boucle=comptage.id_boucle,
datetime=comptage.datetime,
count=comptage.count,
week_day=comptage.week_day,
holiday=comptage.holiday,
)
else: else:
db_comptage = models.Comptage(id_boucle = comptage.id_boucle, datetime = comptage.datetime, count = comptage.count, week_day = comptage.week_day, holiday = comptage.holiday) db_comptage = models.Comptage(
id_boucle=comptage.id_boucle,
datetime=comptage.datetime,
count=comptage.count,
week_day=comptage.week_day,
holiday=comptage.holiday,
)
session.add(db_comptage) session.add(db_comptage)
session.commit() session.commit()
return True return True
def is_boucle_id_exist(id): def is_boucle_id_exist(id):
with Session() as session: with Session() as session:
return session.query(models.Boucle.id).filter_by(id=id).first() is not None return (
session.query(models.Boucle.id).filter_by(id=id).first()
is not None
)
def is_comptage_exist(date, id_boucle, unaccounted_table): def is_comptage_exist(date, id_boucle, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None return (
session.query(models.unaccounted_Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
is not None
)
else: else:
return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() is not None return (
session.query(models.Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
is not None
)
def get_comptage_by_date_and_boucle(date, id_boucle, unaccounted_table): def get_comptage_by_date_and_boucle(date, id_boucle, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
return session.query(models.unaccounted_Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() return (
session.query(models.unaccounted_Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
)
else: else:
return session.query(models.Comptage).filter_by(datetime=date, id_boucle=id_boucle).first() return (
session.query(models.Comptage)
.filter_by(datetime=date, id_boucle=id_boucle)
.first()
)
def update_comptage(comptage: schemas.ComptageBase, unaccounted_table): def update_comptage(comptage: schemas.ComptageBase, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
stmt = update(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count) stmt = (
update(models.unaccounted_Comptage)
.where(models.unaccounted_Comptage.id == comptage.id)
.values(
holiday=comptage.holiday,
week_day=comptage.week_day,
count=comptage.count,
)
)
else: else:
stmt = update(models.Comptage).where(models.Comptage.id == comptage.id).values(holiday=comptage.holiday, week_day=comptage.week_day, count=comptage.count) stmt = (
update(models.Comptage)
.where(models.Comptage.id == comptage.id)
.values(
holiday=comptage.holiday,
week_day=comptage.week_day,
count=comptage.count,
)
)
session.execute(stmt) session.execute(stmt)
session.commit() session.commit()
return True return True
def delete_comptage(comptage_id, unaccounted_table): def delete_comptage(comptage_id, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
stmt = delete(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage_id) stmt = delete(models.unaccounted_Comptage).where(
models.unaccounted_Comptage.id == comptage_id
)
else: else:
stmt = delete(models.Comptage).where(models.Comptage.id == comptage_id) stmt = delete(models.Comptage).where(
models.Comptage.id == comptage_id
)
session.execute(stmt) session.execute(stmt)
session.commit() session.commit()
return True return True
def select_comptage(id_boucle, unaccounted_table): def select_comptage(id_boucle, unaccounted_table):
with Session() as session: with Session() as session:
if unaccounted_table: if unaccounted_table:
stmt = select(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle) stmt = select(models.unaccounted_Comptage).where(
models.unaccounted_Comptage.id_boucle == id_boucle
)
else: else:
stmt = select(models.Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle) stmt = select(models.Comptage).where(
return session.execute(stmt, execution_options={"prebuffer_rows": True}).scalars().all() models.unaccounted_Comptage.id_boucle == id_boucle
)
return (
session.execute(stmt, execution_options={"prebuffer_rows": True})
.scalars()
.all()
)

7
src/sql/database.py
View File

@ -2,11 +2,14 @@ from sqlalchemy import create_engine
from logger import Logger from logger import Logger
from settings import Settings from settings import Settings
settings=Settings() settings = Settings()
db = settings.db db = settings.db
class Database: class Database:
engine = None engine = None
def __init__(self): def __init__(self):
self.engine = create_engine(f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}') self.engine = create_engine(
f'postgresql+psycopg2://{db["user"]}:{db["password"]}@{db["host"]}:{db["port"]}/{db["dbname"]}'
)

26
src/sql/models.py
View File

@ -1,33 +1,43 @@
from sqlalchemy import Column, Integer, String, DateTime, ForeignKey, func, Boolean from sqlalchemy import (
Column,
Integer,
String,
DateTime,
ForeignKey,
func,
Boolean,
)
from sqlalchemy.orm import declarative_base from sqlalchemy.orm import declarative_base
from geoalchemy2 import Geometry from geoalchemy2 import Geometry
Base = declarative_base() Base = declarative_base()
class Boucle(Base): class Boucle(Base):
__tablename__ = 'boucles' __tablename__ = "boucles"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
name = Column(String) name = Column(String)
geolocalisation = Column(Geometry('POINT')) geolocalisation = Column(Geometry("POINT"))
class Comptage(Base): class Comptage(Base):
__tablename__ = 'comptages' __tablename__ = "comptages"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
id_boucle = Column(Integer, ForeignKey("boucles.id")) id_boucle = Column(Integer, ForeignKey("boucles.id"))
datetime = Column('date', DateTime(timezone=True), default=func.now()) datetime = Column("date", DateTime(timezone=True), default=func.now())
week_day = Column(Integer) week_day = Column(Integer)
holiday = Column(Boolean) holiday = Column(Boolean)
count = Column(Integer) count = Column(Integer)
class unaccounted_Comptage(Base): class unaccounted_Comptage(Base):
__tablename__ = 'unaccounted_comptage' __tablename__ = "unaccounted_comptage"
id = Column(Integer, primary_key=True) id = Column(Integer, primary_key=True)
id_boucle = Column(Integer) id_boucle = Column(Integer)
datetime = Column('date', DateTime(timezone=True), default=func.now()) datetime = Column("date", DateTime(timezone=True), default=func.now())
week_day = Column(Integer) week_day = Column(Integer)
holiday = Column(Boolean) holiday = Column(Boolean)
count = Column(Integer) count = Column(Integer)

6
src/sql/schemas.py
View File

@ -1,20 +1,24 @@
from datetime import datetime from datetime import datetime
from pydantic import BaseModel from pydantic import BaseModel
class Boucle(BaseModel): class Boucle(BaseModel):
id: int id: int
name: str name: str
geolocalisation: str geolocalisation: str
class ComptageBase(BaseModel): class ComptageBase(BaseModel):
id_boucle: int id_boucle: int
datetime: datetime datetime: datetime
count: int count: int
week_day = int week_day = int
holiday = bool holiday = bool
class ComptageCreate(ComptageBase): class ComptageCreate(ComptageBase):
pass pass
class Comptage(ComptageBase): class Comptage(ComptageBase):
id: int id: int