diff options
author | Navan Chauhan <navanchauhan@Navans-Mac-mini.local> | 2022-05-20 21:03:04 -0600 |
---|---|---|
committer | Navan Chauhan <navanchauhan@Navans-Mac-mini.local> | 2022-05-20 21:03:04 -0600 |
commit | b8501eac7723510751132a9e71effe99965bd5df (patch) | |
tree | 0c0589af9861c54bd327728f94ef632f8897143a | |
parent | 8395daeec653f0b30f5fb2531bbe1423687d1ed2 (diff) |
updated
-rw-r--r-- | main.py | 5 |
1 files changed, 3 insertions, 2 deletions
@@ -79,13 +79,14 @@ engine, Session = init_db_stuff(database_url) start_time = datetime.now() -for page in tqdm(range(2990,max_requests+10)): +for page in tqdm(range(1,max_requests+10)): if req_count == 999: seconds_to_sleep = 300 - (datetime.now() - start_time).seconds if seconds_to_sleep < 1: - seconds_to_sleep = 100 + seconds_to_sleep = 60 print(f"Sleeping {seconds_to_sleep}s") # Need to respect their rate limitting + # Better to use x-ratelimit header time.sleep(seconds_to_sleep) start_time = datetime.now() req_count = 0 |