From b8501eac7723510751132a9e71effe99965bd5df Mon Sep 17 00:00:00 2001 From: Navan Chauhan Date: Fri, 20 May 2022 21:03:04 -0600 Subject: updated --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'main.py') diff --git a/main.py b/main.py index 1030a62..e2c8adb 100644 --- a/main.py +++ b/main.py @@ -79,13 +79,14 @@ engine, Session = init_db_stuff(database_url) start_time = datetime.now() -for page in tqdm(range(2990,max_requests+10)): +for page in tqdm(range(1,max_requests+10)): if req_count == 999: seconds_to_sleep = 300 - (datetime.now() - start_time).seconds if seconds_to_sleep < 1: - seconds_to_sleep = 100 + seconds_to_sleep = 60 print(f"Sleeping {seconds_to_sleep}s") # Need to respect their rate limitting + # Better to use x-ratelimit header time.sleep(seconds_to_sleep) start_time = datetime.now() req_count = 0 -- cgit v1.2.3