442 lines
16 KiB
Python
Executable File
442 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import argparse
|
|
import logging
|
|
import coloredlogs
|
|
import requests
|
|
import os
|
|
import json
|
|
import zipfile
|
|
import datetime
|
|
import dateutil.parser
|
|
import csv
|
|
import shutil
|
|
import glob
|
|
import sys
|
|
import shutil
|
|
|
|
if not sys.version_info >= (3, 8):
|
|
print("Python >= 3.8 is required")
|
|
exit(1)
|
|
|
|
URL_API_GTFS = "https://data.explore.star.fr/api/records/1.0/search?apikey=3324de385f6b7b5bdfa1727d3e1a9f4bed5dd1accb288db6df5e8487&dataset=tco-busmetro-horaires-gtfs-versions-td&q=&sort=-debutvalidite"
|
|
URL_API_PICTO_BUS = "https://data.explore.star.fr/api/records/1.0/search/?apikey=3324de385f6b7b5bdfa1727d3e1a9f4bed5dd1accb288db6df5e8487&dataset=tco-bus-lignes-pictogrammes-dm&q=&rows=200&sort=idligne&refine.resolution=1%3A22"
|
|
URL_API_PICTO_METRO = "https://data.explore.star.fr/api/records/1.0/search/?apikey=3324de385f6b7b5bdfa1727d3e1a9f4bed5dd1accb288db6df5e8487&dataset=tco-metro-lignes-pictogrammes-dm&q=&rows=200&sort=idligne&refine.resolution=1%3A22"
|
|
URL_PICTO_PREFIX_BUS = (
|
|
"https://data.explore.star.fr/explore/dataset/tco-bus-lignes-pictogrammes-dm/files/"
|
|
)
|
|
URL_PICTO_PREFIX_METRO = "https://data.explore.star.fr/explore/dataset/tco-metro-lignes-pictogrammes-dm/files/"
|
|
|
|
|
|
class Station(object):
|
|
"""
|
|
Classe qui représente une station.
|
|
Utilisé pour transformer un object en json
|
|
"""
|
|
|
|
def __init__(self, sid, name=None, lon=None, lat=None):
|
|
self.lat = lat
|
|
self.lon = lon
|
|
self.sid = sid
|
|
self.name = name
|
|
self.terminus = None
|
|
self.limits = None
|
|
self.lines = None
|
|
self.parcours = None
|
|
|
|
def __str__(self):
|
|
o = {}
|
|
o["type"] = "Feature"
|
|
o["geometry"] = {}
|
|
o["geometry"]["type"] = "Point"
|
|
o["geometry"]["coordinates"] = [self.lon, self.lat]
|
|
o["properties"] = {}
|
|
o["properties"]["id"] = self.sid
|
|
o["properties"]["name"] = self.name
|
|
o["properties"]["terminus"] = list(self.terminus)
|
|
o["properties"]["limits"] = self.limits
|
|
o["properties"]["lines"] = list(self.lines)
|
|
o["properties"]["parcours"] = list(self.parcours)
|
|
return json.dumps(o)
|
|
|
|
def __repr__(self):
|
|
return self.__str__()
|
|
|
|
|
|
def get_gtfs_dir(today):
|
|
"""
|
|
Récupère le repertoire GTFS le plus récent (en fonction de la publication)
|
|
qui correspond à la date passé en paramètre.
|
|
Si il n'y a pas de de données GTFS disponible retourne None.
|
|
"""
|
|
gtfs_path = "./GTFS/"
|
|
available_dir = []
|
|
for filename in os.listdir(gtfs_path):
|
|
logger.debug("Found directory: " + filename)
|
|
gtfs_dir = gtfs_path + filename
|
|
if os.path.isdir(gtfs_dir):
|
|
start = datetime.datetime.strptime(
|
|
filename.split("_")[0], "%Y-%m-%d"
|
|
).date()
|
|
end = datetime.datetime.strptime(filename.split("_")[1], "%Y-%m-%d").date()
|
|
if today >= start and today <= end:
|
|
available_dir.append(gtfs_dir)
|
|
available_dir_sorted = sorted(available_dir, key=lambda e: e.split("_")[2])
|
|
if len(available_dir_sorted) > 0:
|
|
gtfs_dir = available_dir_sorted[-1]
|
|
logger.debug("Use directory: " + gtfs_dir)
|
|
return gtfs_dir
|
|
else:
|
|
logger.debug("No GTFS data for date: " + str(today))
|
|
return
|
|
|
|
|
|
#### Generateur de données ###
|
|
# Recodé en python depuis le générateur original en nodejs
|
|
# Ajout des parcours en plus pour chaque station
|
|
|
|
|
|
def get_serviceid_today(gtfs_dir, today):
|
|
services = []
|
|
csv_calendar = list(csv.DictReader(open(gtfs_dir + "/calendar.txt")))
|
|
weekdays = list(csv_calendar[0].keys())[1:8]
|
|
for i in csv_calendar:
|
|
start_date = datetime.datetime.strptime(i["start_date"], "%Y%m%d").date()
|
|
end_date = datetime.datetime.strptime(i["end_date"], "%Y%m%d").date()
|
|
weekday = weekdays[today.weekday()]
|
|
if i[weekday] == "1" and today >= start_date and today <= end_date:
|
|
services.append(i["service_id"])
|
|
|
|
csv_calendar_dates = list(csv.DictReader(open(gtfs_dir + "/calendar_dates.txt")))
|
|
for i in csv_calendar_dates:
|
|
c_date = datetime.datetime.strptime(i["date"], "%Y%m%d").date()
|
|
if today == c_date:
|
|
services.append(i["service_id"])
|
|
|
|
return services
|
|
|
|
|
|
def get_trips_from_services(gtfs_dir, services_today):
|
|
trips = {}
|
|
csv_trips = list(csv.DictReader(open(gtfs_dir + "/trips.txt")))
|
|
for i in csv_trips:
|
|
if i["service_id"] in services_today:
|
|
info = {}
|
|
info["route_id"] = i["route_id"]
|
|
info["shape_id"] = i["shape_id"]
|
|
info["sens"] = i["direction_id"]
|
|
info["last_sequence"] = "1"
|
|
trips[i["trip_id"]] = info
|
|
|
|
return trips
|
|
|
|
|
|
def generate_stop_map(gtfs_dir):
|
|
all_stops = {}
|
|
|
|
csv_stops = list(csv.DictReader(open(gtfs_dir + "/stops.txt")))
|
|
for i in csv_stops:
|
|
stop_detail = {}
|
|
stop_detail["id"] = i["stop_id"]
|
|
stop_detail["name"] = i["stop_name"]
|
|
stop_detail["lat"] = i["stop_lat"]
|
|
stop_detail["lon"] = i["stop_lon"]
|
|
stop_detail["trips"] = {}
|
|
all_stops[i["stop_id"]] = stop_detail
|
|
|
|
return all_stops
|
|
|
|
|
|
def associate_trips(gtfs_dir, all_stops, trips):
|
|
|
|
csv_stop_times = list(csv.DictReader(open(gtfs_dir + "/stop_times.txt")))
|
|
for i in csv_stop_times:
|
|
if i["trip_id"] in trips:
|
|
stop_info = {}
|
|
stop_info["arrival_time"] = i["arrival_time"]
|
|
stop_info["departure_time"] = i["departure_time"]
|
|
stop_info["route_id"] = trips[i["trip_id"]]["route_id"]
|
|
stop_info["shape_id"] = trips[i["trip_id"]]["shape_id"]
|
|
stop_info["sens"] = trips[i["trip_id"]]["sens"]
|
|
trips[i["trip_id"]]["last_sequence"] = str(
|
|
max(int(trips[i["trip_id"]]["last_sequence"]), int(i["stop_sequence"]))
|
|
)
|
|
stop_info["stop_sequence"] = i["stop_sequence"]
|
|
all_stops[i["stop_id"]]["trips"][i["trip_id"]] = stop_info
|
|
|
|
return all_stops, trips
|
|
|
|
|
|
def filter_no_trips(all_stops):
|
|
newDict = {}
|
|
for (k, v) in all_stops.items():
|
|
if len(all_stops[k]["trips"]) != 0:
|
|
newDict[k] = v
|
|
return newDict
|
|
|
|
|
|
def finalize_stops(all_stops, trips):
|
|
all_stops = filter_no_trips(all_stops)
|
|
all_stations = []
|
|
for stop_key, stop_value in all_stops.items():
|
|
station = Station(stop_key)
|
|
station.lat = stop_value["lat"]
|
|
station.lon = stop_value["lon"]
|
|
station.name = stop_value["name"]
|
|
terminus_dict = {}
|
|
station.terminus = set()
|
|
first_trip = next(iter((stop_value["trips"].items())))
|
|
first_arrival_time = first_trip[1]["arrival_time"]
|
|
last_departure_time = first_trip[1]["departure_time"]
|
|
station.lines = set()
|
|
station.parcours = set()
|
|
for (k, v) in stop_value["trips"].items():
|
|
if v["stop_sequence"] == trips[k]["last_sequence"]:
|
|
terminus_dict[k] = v
|
|
arrival_time = v["arrival_time"]
|
|
departure_time = v["departure_time"]
|
|
first_arrival_time = min(first_arrival_time, arrival_time)
|
|
last_departure_time = max(last_departure_time, departure_time)
|
|
station.lines.add(v["route_id"] + "," + v["sens"])
|
|
if len(v["shape_id"]):
|
|
station.parcours.add(v["shape_id"])
|
|
for (k, v) in terminus_dict.items():
|
|
station.terminus.add(v["route_id"] + "," + v["sens"])
|
|
station.limits = [first_arrival_time, last_departure_time]
|
|
all_stations.append(station)
|
|
return all_stations
|
|
|
|
|
|
### Fin du générateur
|
|
|
|
|
|
def stations():
|
|
"""
|
|
Génère toutes les données geojson possible depuis aujourd'hui
|
|
jusqu'à la fin des données GTFS disponible.
|
|
Si les données des stations ont déjà été generées et qu'elles proviennt de données
|
|
GTFS qui ont la même date de publication on ne les génère pas à nouveau.
|
|
"""
|
|
|
|
os.makedirs("./GTFS/", exist_ok=True)
|
|
os.makedirs("./data/", exist_ok=True)
|
|
today = datetime.datetime.now().date()
|
|
while gtfs_dir := get_gtfs_dir(today):
|
|
logger.debug("Generate data for " + str(today))
|
|
publication_gtfs = gtfs_dir.split("_")[-1]
|
|
publication_gtfs_date = datetime.datetime.fromtimestamp(
|
|
int(publication_gtfs)
|
|
).date()
|
|
logger.debug("Use %s (%s)", gtfs_dir, str(publication_gtfs_date))
|
|
filename = "./data/" + today.strftime("stations_%Y_%m_%d.geojson")
|
|
if os.path.exists(filename):
|
|
logger.debug(filename + " already exists")
|
|
with open(filename + ".info", "r") as f:
|
|
present_file_info = json.load(f)
|
|
present_file_publication = present_file_info["publication"]
|
|
logger.debug(
|
|
"File is generated from data published on " + present_file_publication
|
|
)
|
|
present_file_publication_date = datetime.datetime.fromtimestamp(
|
|
int(present_file_publication)
|
|
).date()
|
|
if present_file_publication_date == publication_gtfs_date:
|
|
logger.debug("stations file is up to date")
|
|
today += datetime.timedelta(days=1)
|
|
continue
|
|
services_today = get_serviceid_today(gtfs_dir, today)
|
|
trips = get_trips_from_services(gtfs_dir, services_today)
|
|
all_stops = generate_stop_map(gtfs_dir)
|
|
all_stops, trips = associate_trips(gtfs_dir, all_stops, trips)
|
|
all_stations = finalize_stops(all_stops, trips)
|
|
with open(filename, "w") as f:
|
|
f.write(str(all_stations))
|
|
with open(filename + ".info", "w") as f:
|
|
info = {"publication": publication_gtfs}
|
|
json.dump(info, f)
|
|
today += datetime.timedelta(days=1)
|
|
# import ipdb; ipdb.set_trace()
|
|
|
|
|
|
def download_file(url, output_filename):
|
|
"""
|
|
Helper: télécharge un fichier
|
|
"""
|
|
logger.info("Download: %s", url)
|
|
with requests.get(url, stream=True) as r:
|
|
r.raise_for_status()
|
|
with open(output_filename, "wb") as f:
|
|
for chunk in r.iter_content(chunk_size=8192):
|
|
# If you have chunk encoded response uncomment if
|
|
# and set chunk_size parameter to None.
|
|
# if chunk:
|
|
f.write(chunk)
|
|
|
|
|
|
def picto():
|
|
"""
|
|
Télécharge tous les picto disponibles.
|
|
Si ils sont déjà présent on ne fait rien.
|
|
"""
|
|
|
|
for param in [
|
|
(URL_API_PICTO_BUS, URL_PICTO_PREFIX_BUS),
|
|
(URL_API_PICTO_METRO, URL_PICTO_PREFIX_METRO),
|
|
]:
|
|
r = requests.get(param[0])
|
|
data = r.json()
|
|
records = data["records"]
|
|
logger.debug(json.dumps(records, indent=4))
|
|
for entry in records:
|
|
filename = "./images/picto_" + entry["fields"]["idligne"] + ".png"
|
|
if not os.path.exists(filename):
|
|
url = param[1] + entry["fields"]["image"]["id"] + "/download/"
|
|
download_file(url, filename)
|
|
assert os.path.exists(filename)
|
|
|
|
|
|
def download():
|
|
"""
|
|
Télécharge les fichier zip des données GTFS puis les extrait.
|
|
Si le fichier est déjà téléchargé on ne fait rien.
|
|
"""
|
|
|
|
os.makedirs("./downloads/", exist_ok=True)
|
|
os.makedirs("./GTFS/", exist_ok=True)
|
|
r = requests.get(URL_API_GTFS)
|
|
data = r.json()
|
|
records = data["records"]
|
|
logger.debug(json.dumps(records, indent=4))
|
|
for entry in records:
|
|
filename = "./downloads/" + entry["fields"]["fichier"]["filename"]
|
|
filename_info = filename + ".json"
|
|
with open(filename_info, "w") as f:
|
|
f.write(json.dumps(entry, indent=4))
|
|
size = entry["fields"]["tailleoctets"]
|
|
if not os.path.exists(filename) or os.path.getsize(filename) != size:
|
|
url = entry["fields"]["url"]
|
|
output_filename = "./downloads/" + url.split("/")[-1]
|
|
download_file(url, output_filename)
|
|
assert os.path.exists(filename)
|
|
assert os.path.getsize(filename) == size
|
|
publication_timestamp = dateutil.parser.parse(
|
|
entry["fields"]["publication"]
|
|
).timestamp()
|
|
path_extract = (
|
|
"./GTFS/"
|
|
+ entry["fields"]["debutvalidite"]
|
|
+ "_"
|
|
+ entry["fields"]["finvalidite"]
|
|
+ "_"
|
|
+ str(int(publication_timestamp))
|
|
)
|
|
if not os.path.exists(path_extract):
|
|
os.mkdir(path_extract)
|
|
if not os.listdir(path_extract):
|
|
with zipfile.ZipFile(filename) as zf:
|
|
zf.extractall(path_extract)
|
|
|
|
|
|
def clean():
|
|
"""
|
|
Supprime toutes les données produites par ce programme.
|
|
"""
|
|
|
|
shutil.rmtree("./downloads")
|
|
shutil.rmtree("./GTFS")
|
|
for picto in glob.glob("./images/picto_*"):
|
|
os.remove(picto)
|
|
for stations in glob.glob("./data/stations_*"):
|
|
os.remove(stations)
|
|
|
|
|
|
def prune():
|
|
"""
|
|
Supprime les fichier zip, les données GTFS et les stations si elles ne sont
|
|
plus valide (avec un delta de deux jours).
|
|
"""
|
|
|
|
today = (datetime.datetime.now() - datetime.timedelta(days=2)).date()
|
|
for zipfile in glob.glob("./downloads/*.zip"):
|
|
zipfileinfo = zipfile + ".json"
|
|
with open(zipfileinfo, "r") as f:
|
|
info = json.load(f)
|
|
fin = dateutil.parser.parse(info["fields"]["finvalidite"]).date()
|
|
if today > fin:
|
|
os.remove(zipfile) if os.path.exists(zipfile) else None
|
|
os.remove(zipfileinfo) if os.path.exists(zipfileinfo) else None
|
|
logger.debug(zipfile + " removed")
|
|
logger.debug(zipfileinfo + " removed")
|
|
for gtfsdir in os.listdir("./GTFS/"):
|
|
fin = dateutil.parser.parse(gtfsdir.split("_")[1]).date()
|
|
if today > fin:
|
|
dir_to_remove = "./GTFS/" + gtfsdir
|
|
shutil.rmtree(dir_to_remove)
|
|
logger.debug(dir_to_remove + " removed")
|
|
for stationsfile in glob.glob("./data/stations_*"):
|
|
filename = os.path.basename(stationsfile)
|
|
datefromfile = "-".join(filename.split(".")[:1][0].split("_")[1:])
|
|
date = dateutil.parser.parse(datefromfile).date()
|
|
if today > date:
|
|
os.remove(stationsfile) if os.path.exists(stationsfile) else None
|
|
os.remove(stationsfile + ".info") if os.path.exists(
|
|
stationsfile + ".info"
|
|
) else None
|
|
logger.debug(stationsfile + "removed")
|
|
|
|
|
|
def auto():
|
|
"""
|
|
Execute toutes les tâches nécéssaire au fonctionnement du service.
|
|
Commande à executer chaque jour avec tâche cron.
|
|
"""
|
|
|
|
picto()
|
|
download()
|
|
stations()
|
|
prune()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# fmt: off
|
|
description_str = ('ouestmonbus.com GTFS downloader \n' +
|
|
'and data generator\n')
|
|
# fmt: on
|
|
usage_str = "%(prog)s action"
|
|
epilog_str = "v 0.1"
|
|
frm = argparse.RawTextHelpFormatter
|
|
parser = argparse.ArgumentParser(
|
|
description=description_str,
|
|
epilog=epilog_str,
|
|
usage=usage_str,
|
|
formatter_class=frm,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--log",
|
|
help="log level",
|
|
default="INFO",
|
|
)
|
|
cmd_parser = parser.add_subparsers(dest="command", help="command parser")
|
|
subparser = cmd_parser.add_parser("download", help="download GTFS")
|
|
subparser = cmd_parser.add_parser("stations", help="generate stations geojson")
|
|
subparser = cmd_parser.add_parser("picto", help="download picto")
|
|
subparser = cmd_parser.add_parser("clean", help="delete all")
|
|
subparser = cmd_parser.add_parser("prune", help="delete expired data")
|
|
subparser = cmd_parser.add_parser(
|
|
"auto", help="execute all needed task to use with cron"
|
|
)
|
|
subparser.add_argument(
|
|
"--days",
|
|
type=int,
|
|
default=1,
|
|
help="nb days to generate",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
coloredlogs.install(level=args.log)
|
|
logger = logging.getLogger(__name__)
|
|
os.chdir(os.path.dirname(__file__))
|
|
locals()[args.command]()
|