Merge branch 'feature/update_boucles' into develop

src/main.py

@@ -1,7 +1,6 @@
 import json
 import pendulum
 import requests
-import time
 import typer
 from logger import Logger
 from parser import parseBoucles, parseRecord
@@ -42,6 +41,50 @@ def populate_boucles():
             else:
                 log.debug("Records is empty, stop it")
                 break
+    log.info("Check if new boucles added and move records into comptages table")
+    for id_boucle in crud.get_unaccounted_record_with_exist_boucle():
+        log.info(f'New boucle detected. ID {id_boucle}')
+        log.info('Moving record from unaccounted_comptage to comptage')
+        for record in crud.select_comptage(id_boucle, True):
+            log.info(f'Moving record with ID {record.id}')
+            crud.create_comptage(record, False)
+            log.info(f'Delete record with ID {record.id}')
+            crud.delete_comptage(record.id, True)
+
+@app.command()
+def prepopulate_today():
+    today = pendulum.today("UTC")
+
+    limit = 10
+    start = 0
+    process = True
+    while process:
+        r = requests.get(
+            f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b"
+        )
+        if r.status_code == requests.codes.ok:
+            data = r.json()
+            if data["records"] != []:
+                for record in data["records"]:
+                    log.debug(f'record date is {record["fields"]["dateformat"]}')
+                    record_date = pendulum.parse(record["fields"]["dateformat"])
+                    if record_date == today:
+                        log.info("Date is good")
+                        parseRecord(record)
+                    else:
+                        log.info("Stop processing")
+                        process = False
+                        break
+                start += 10
+            else:
+                log.debug("Records is empty, stop it")
+                process = False
+                pprint.pprint(data["records"])
+
+        else:
+            pprint.pprint(r.status_code)
+            pprint.pprint(r.text)
+            process = False
 
 
 @app.command()

src/parser.py

@@ -15,12 +15,22 @@ log = logging.log
 def parseBoucles(boucles):
     for boucle in boucles:
         log.info(f'Get {boucle["fields"]["libelle"]}')
-        boucleModel = models.Boucle(
-            id=boucle["fields"]["boucle_num"],
-            name=boucle["fields"]["libelle"],
-            geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})'
-        )
-        crud.create_boucle(boucleModel)
+        if not crud.is_boucle_id_exist(boucle["fields"]["boucle_num"]):
+            log.info(f'Insert {boucle["fields"]["libelle"]}')
+            boucleModel = models.Boucle(
+                id=boucle["fields"]["boucle_num"],
+                name=boucle["fields"]["libelle"],
+                geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})'
+            )
+            crud.create_boucle(boucleModel)
+        else:
+            log.info(f'Update {boucle["fields"]["libelle"]}')
+            boucleModel = models.Boucle(
+                id=boucle["fields"]["boucle_num"],
+                name=boucle["fields"]["libelle"],
+                geolocalisation=f'POINT({boucle["fields"]["geolocalisation"][0]} {boucle["fields"]["geolocalisation"][1]})'
+            )
+            crud.update_boucle(boucleModel)
 
 def parseRecord(record, force = False):
     if not crud.is_boucle_id_exist(record["fields"]["boucle_num"]):

src/settings.py

@@ -4,7 +4,6 @@ from pathlib import Path
 
 class Settings:
     db = {}
-    logging = {}
 
 
     def __init__(self):
@@ -14,6 +13,5 @@ class Settings:
             try:
                 config = yaml.safe_load(stream)
                 self.db = config["db"]
-                self.logging = config["logging"]
             except yaml.YAMLError as exc:
                 print(exc)

src/sql/crud.py

@@ -1,6 +1,6 @@
 from . import models, schemas
 from .database import Database
-from sqlalchemy import update
+from sqlalchemy import delete, update, select, text
 from sqlalchemy.orm import sessionmaker
 
 import pprint
@@ -15,6 +15,18 @@ def create_boucle(boucle: schemas.Boucle):
         session.commit()
         return True 
 
+def update_boucle(boucle: schemas.Boucle):
+    with Session() as session:
+        stmt = update(models.Boucle).where(models.Boucle.id == boucle.id).values(name=boucle.name, geolocalisation=boucle.geolocalisation)
+        session.execute(stmt)
+        return True
+
+def get_unaccounted_record_with_exist_boucle():
+    with Session() as session:
+        textual_sql = text("SELECT id_boucle FROM unaccounted_comptage WHERE EXISTS (SELECT id FROM boucles WHERE id = unaccounted_comptage.id_boucle) GROUP BY id_boucle;")
+        return session.execute(textual_sql).scalars()
+
+
 def create_comptage(comptage: schemas.ComptageBase, unaccounted_table):
     with Session() as session:
         if unaccounted_table:
@@ -50,4 +62,24 @@ def update_comptage(comptage: schemas.ComptageBase, unaccounted_table):
         else:
             stmt = update(models.Comptage).where(models.Comptage.id == comptage.id).values(count=comptage.count)
         session.execute(stmt)
+        session.commit()
         return True
+
+def delete_comptage(comptage_id, unaccounted_table):
+    with Session() as session:
+        if unaccounted_table:
+            stmt = delete(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id == comptage_id)
+        else:
+            stmt = delete(models.Comptage).where(models.Comptage.id == comptage_id)
+        session.execute(stmt)
+        session.commit()
+        return True
+
+def select_comptage(id_boucle, unaccounted_table):
+    with Session() as session:
+        if unaccounted_table:
+            stmt = select(models.unaccounted_Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle)
+        else:
+            stmt = select(models.Comptage).where(models.unaccounted_Comptage.id_boucle == id_boucle)
+        return session.execute(stmt, execution_options={"prebuffer_rows": True}).scalars().all()
+