141 lines
5.0 KiB
Python
141 lines
5.0 KiB
Python
import json
|
|
import pendulum
|
|
import requests
|
|
import time
|
|
import typer
|
|
from logger import Logger
|
|
from parser import parseBoucles, parseRecord
|
|
from settings import Settings
|
|
from sql import crud, models, database
|
|
|
|
import pprint
|
|
|
|
settings = Settings()
|
|
logging = Logger(__name__)
|
|
log = logging.log
|
|
|
|
app = typer.Typer()
|
|
|
|
|
|
@app.command()
|
|
def create_db():
|
|
db = database.Database()
|
|
models.Boucle.__table__.create(db.engine)
|
|
models.Comptage.__table__.create(db.engine)
|
|
models.unaccounted_Comptage.__table__.create(db.engine)
|
|
|
|
|
|
@app.command()
|
|
def populate_boucles():
|
|
limit = 10
|
|
start = 0
|
|
while True:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole-boucles-comptage&q=&rows={limit}&start={start}&sort=boucle_num&facet=boucle_num&facet=libelle"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
log.info("Records is not empty, increase start from 10")
|
|
start += 10
|
|
parseBoucles(data["records"])
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
break
|
|
|
|
|
|
@app.command()
|
|
def populate_day():
|
|
today = pendulum.today("UTC")
|
|
# today = pendulum.datetime(2022, 6, 16, tz='UTC')
|
|
yesterday = pendulum.yesterday("UTC")
|
|
# yesterday = pendulum.datetime(2022, 6, 15, tz='UTC')
|
|
good_day = today.subtract(days=2)
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(f'record date is {record["fields"]["dateformat"]}')
|
|
record_date = pendulum.parse(record["fields"]["dateformat"])
|
|
if record_date == today or record_date == yesterday:
|
|
log.info("Date is not complete, ignore it")
|
|
elif record_date == good_day:
|
|
log.info("Date is good")
|
|
parseRecord(record)
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
pprint.pprint(data["records"])
|
|
|
|
else:
|
|
pprint.pprint(r.status_code)
|
|
pprint.pprint(r.text)
|
|
process = False
|
|
|
|
|
|
@app.command()
|
|
def populate_all(file: str, force: bool = typer.Option(False, help="Force update")):
|
|
with open(f"{file}", 'r') as json_file:
|
|
data = json.load(json_file)
|
|
for record in data:
|
|
log.debug(f'record date is {record["fields"]["dateformat"]}')
|
|
record_date = pendulum.parse(record["fields"]["dateformat"])
|
|
parseRecord(record, force)
|
|
|
|
|
|
@app.command()
|
|
def consolidate_week():
|
|
today = pendulum.today("UTC")
|
|
yesterday = pendulum.yesterday("UTC")
|
|
good_day_start = today.subtract(days=2)
|
|
good_day_end = today.subtract(days=9)
|
|
|
|
limit = 10
|
|
start = 0
|
|
process = True
|
|
while process:
|
|
r = requests.get(
|
|
f"https://data.nantesmetropole.fr/api/records/1.0/search/?dataset=244400404_comptages-velo-nantes-metropole&q=&rows={limit}&start={start}&sort=jour&facet=boucle_num&facet=libelle&facet=jour&facet=probabilite_presence_anomalie&facet=jour_de_la_semaine&facet=boucle_libelle&facet=vacances_zone_b"
|
|
)
|
|
if r.status_code == requests.codes.ok:
|
|
data = r.json()
|
|
if data["records"] != []:
|
|
for record in data["records"]:
|
|
log.debug(f'record date is {record["fields"]["dateformat"]}')
|
|
record_date = pendulum.parse(record["fields"]["dateformat"])
|
|
if record_date == today or record_date == yesterday:
|
|
log.info("Date is not complete, ignore it")
|
|
elif good_day_end <= record_date <= good_day_start:
|
|
log.info("Date is good")
|
|
parseRecord(record)
|
|
else:
|
|
log.info("Stop processing")
|
|
process = False
|
|
break
|
|
start += 10
|
|
else:
|
|
log.debug("Records is empty, stop it")
|
|
process = False
|
|
pprint.pprint(data["records"])
|
|
else:
|
|
pprint.pprint(r.status_code)
|
|
pprint.pprint(r.text)
|
|
process = False
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app()
|