undefined_name/sncb_crawl
5
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

.gitignore + safety header

Browse Source
This commit is contained in:
Debucquoy Anthony
2022-08-28 16:26:50 +02:00
parent 33cf0cf461
commit ac23c63ae5
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
env/

6
main.py
View File

@ -27,6 +27,8 @@ async def main():
async def fetch_station(station: str, session: aiohttp.ClientSession, trains: list) -> None: async def fetch_station(station: str, session: aiohttp.ClientSession, trains: list) -> None:
train_list = [] train_list = []
url = "http://www.belgianrail.be/jp/nmbs-realtime/stboard.exe/en" url = "http://www.belgianrail.be/jp/nmbs-realtime/stboard.exe/en"
url_header = {'User-Agent': 'python script to fetch trains of belgium. (d.tonitch@gmail.com - in case it is a problem)',
'From': 'd.tonitch@gmail.com'}
url_data = { url_data = {
'realtime': 'Show', # Dunno 'realtime': 'Show', # Dunno
'sqQueryPageDisplayed': 'yes', # Dunno 'sqQueryPageDisplayed': 'yes', # Dunno
@ -43,7 +45,7 @@ async def fetch_station(station: str, session: aiohttp.ClientSession, trains: li
'start': 'Show' # Dunno 'start': 'Show' # Dunno
} }
try: try:
async with session.post(url, data=url_data) as resp: async with session.post(url, data=url_data, headers=url_header) as resp:
if resp.status == 200: if resp.status == 200:
print(f"station {station} success ✅") print(f"station {station} success ✅")
soup = BeautifulSoup(await resp.text(), 'html.parser') soup = BeautifulSoup(await resp.text(), 'html.parser')
@ -52,7 +54,7 @@ async def fetch_station(station: str, session: aiohttp.ClientSession, trains: li
train_list = [''.join(i.a.contents) for i in products] train_list = [''.join(i.a.contents) for i in products]
for train in train_list: for train in train_list:
if train not in trains and train: if train not in trains and train:
trains.append(re.sub(" +", " ",train.strip())) trains.append(re.sub(" +", " ", train.strip()))
return None return None
except aiohttp.client_exceptions.ServerDisconnectedError: except aiohttp.client_exceptions.ServerDisconnectedError:
print(f"station {station} failed ❎ ") print(f"station {station} failed ❎ ")