agresdominik/predictify
1
0
Fork 0
mirror of https://github.com/agresdominik/predictify.git synced 2026-04-21 17:55:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed counter issue where the accumilation would be calculated wrongly and the requests were not at the fullest capacity

Browse Source
This commit is contained in:
agres
2025-03-23 21:54:44 +01:00
parent 164ea9aca9
commit e73752c015
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scraper.py
+7 -3
View File
@@ -60,21 +60,25 @@ def _scrape_missing_info(bearer_token_simple: str, table_name: Table, id_field_n
ids = [] ids = []
processed_ids = set() processed_ids = set()
for i, id_value in enumerate(all_ids_missing): counter = 0
for id_value in all_ids_missing:
id_value_str = id_value[0] id_value_str = id_value[0]
if id_value_str not in processed_ids: if id_value_str not in processed_ids:
ids.append(id_value_str) ids.append(id_value_str)
processed_ids.add(id_value_str) processed_ids.add(id_value_str)
counter += 1
if (i + 1) % limit == 0: if (counter + 1) % limit == 0 and len(ids) > 0:
ids_tuple = tuple(ids) ids_tuple = tuple(ids)
ids.clear() ids.clear()
response = get_multiple_field_information(bearer_token_simple, endpoint_name, limit, *ids_tuple) response = get_multiple_field_information(bearer_token_simple, endpoint_name, limit, *ids_tuple)
_add_data_to_database(table_name, response) _add_data_to_database(table_name, response)
counter = 0
if ids: if len(ids) > 0:
ids_tuple = tuple(ids) ids_tuple = tuple(ids)
ids.clear() ids.clear()
response = get_multiple_field_information(bearer_token_simple, endpoint_name, limit, *ids_tuple) response = get_multiple_field_information(bearer_token_simple, endpoint_name, limit, *ids_tuple)