239 lines
9 KiB
Python
239 lines
9 KiB
Python
import json
|
|
import pendulum
|
|
import requests
|
|
import typer
|
|
from logger import Logger
|
|
from parser import parseBoucles, parseRecord
|
|
from settings import Settings
|
|
from rich import print
|
|
from sql import crud, models, database
|
|
|
|
settings = Settings()
|
|
logging = Logger(__name__)
|
|
log = logging.log
|
|
|
|
app = typer.Typer()
|
|
|
|
|
|
@app.command()
|
|
def create_db():
|
|
db = database.Database()
|
|
models.Boucle.__table__.create(db.engine)
|
|
models.Comptage.__table__.create(db.engine)
|
|
models.unaccounted_Comptage.__table__.create(db.engine)
|
|
|
|
|
|
@app.command()
|
|
def populate_boucles():
|
|
limit = 10
|
|
start = 0
|
|
while True:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole-boucles-comptage&q=&rows={limit}&start={start}&sort=boucle_num&facet=boucle_num&facet=libelle"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
print(data)
|
|
if data["records"] != []:
|
|
log.info("Records is not empty, increase start from 10")
|
|
start += 10
|
|
parseBoucles(data["records"])
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
break
|
|
log.info(
|
|
"Check if new boucles added and move records into comptages table"
|
|
)
|
|
for id_boucle in crud.get_unaccounted_record_with_exist_boucle():
|
|
log.info(f"New boucle detected. ID {id_boucle}")
|
|
log.info("Moving record from unaccounted_comptage to comptage")
|
|
for record in crud.select_comptage(id_boucle, True):
|
|
log.info(f"Moving record with ID {record.id}")
|
|
crud.create_comptage(record, False)
|
|
log.info(f"Delete record with ID {record.id}")
|
|
crud.delete_comptage(record.id, True)
|
|
|
|
|
|
@app.command()
|
|
def prepopulate_today():
|
|
today = pendulum.today("UTC")
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b&order_by=dateformat%20desc"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(
|
|
f'record date is {record["fields"]["dateformat"]}'
|
|
)
|
|
record_date = pendulum.parse(
|
|
record["fields"]["dateformat"]
|
|
)
|
|
if record_date == today:
|
|
log.info("Date is good")
|
|
parseRecord(record["fields"])
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
|
|
else:
|
|
log.error(f"API return a bad status code : {r.status_code}")
|
|
process = False
|
|
|
|
|
|
@app.command()
|
|
def populate_day():
|
|
today = pendulum.today("UTC")
|
|
yesterday = pendulum.yesterday("UTC")
|
|
good_day = today.subtract(days=2)
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b&order_by=dateformat%20desc"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(
|
|
f'record date is {record["fields"]["dateformat"]}'
|
|
)
|
|
record_date = pendulum.parse(
|
|
record["fields"]["dateformat"]
|
|
)
|
|
if record_date == today or record_date == yesterday:
|
|
log.info("Date is not complete, ignore it")
|
|
elif record_date == good_day:
|
|
log.info("Date is good")
|
|
parseRecord(record["fields"])
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
|
|
else:
|
|
log.error(f"API return a bad status code : {r.status_code}")
|
|
process = False
|
|
|
|
|
|
@app.command()
|
|
def populate_all(
|
|
file: str, force: bool = typer.Option(False, help="Force update")
|
|
):
|
|
with open(f"{file}", "r") as json_file:
|
|
data = json.load(json_file)
|
|
for record in data:
|
|
log.debug(f'record date is {record["dateformat"]}')
|
|
parseRecord(record, force)
|
|
|
|
|
|
@app.command()
|
|
def consolidate_week():
|
|
today = pendulum.today("UTC")
|
|
yesterday = pendulum.yesterday("UTC")
|
|
good_day_start = today.subtract(days=2)
|
|
good_day_end = today.subtract(days=9)
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b&order_by=dateformat%20desc"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(
|
|
f'record date is {record["fields"]["dateformat"]}'
|
|
)
|
|
record_date = pendulum.parse(
|
|
record["fields"]["dateformat"]
|
|
)
|
|
if record_date == today or record_date == yesterday:
|
|
log.info("Date is not complete, ignore it")
|
|
elif good_day_end <= record_date <= good_day_start:
|
|
log.info("Date is good")
|
|
parseRecord(record["fields"])
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
else:
|
|
log.error(f"API return a bad status code : {r.status_code}")
|
|
process = False
|
|
|
|
@app.command()
|
|
def consolidate_month():
|
|
today = pendulum.today("UTC")
|
|
yesterday = pendulum.yesterday("UTC")
|
|
current_day = today.day
|
|
last_month = pendulum.now("UTC").subtract(days=current_day)
|
|
good_day_end = pendulum.datetime(last_month.year, last_month.month, 1, tz='UTC')
|
|
good_day_start = pendulum.datetime(last_month.year, last_month.month, last_month.days_in_month, tz='UTC')
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
log.debug(f"Process date between {good_day_start} and {good_day_end}")
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b&order_by=dateformat%20desc"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(
|
|
f'record date is {record["fields"]["dateformat"]}'
|
|
)
|
|
record_date = pendulum.parse(
|
|
record["fields"]["dateformat"]
|
|
)
|
|
if record_date > good_day_start:
|
|
log.debug(f"{record_date} is more recent than {good_day_end}")
|
|
elif good_day_end <= record_date <= good_day_start:
|
|
log.info("Date is good")
|
|
parseRecord(record["fields"])
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
else:
|
|
log.error(f"API return a bad status code : {r.status_code}")
|
|
process = False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app()
|
|
|
|
if __name__ == "__main__":
|
|
app()
|