diff --git a/measprocess/__init__.py b/measprocess/__init__.py index f619d5035252b21d41a6422c177263473cf6867a..04f65a174de8664333fb60246ed2d3eebb2ca89e 100644 --- a/measprocess/__init__.py +++ b/measprocess/__init__.py @@ -4,3 +4,5 @@ from .geospatial import get_geoseries_streets, get_geoseries_blockages from .geospatial import project_onto_streets from .plotting import plot_series_osm + +from .data_extractor import fetch_rtr_details diff --git a/measprocess/data_extractor.py b/measprocess/data_extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..670b482e9f810c17279656f97448682839cf8f03 --- /dev/null +++ b/measprocess/data_extractor.py @@ -0,0 +1,44 @@ +import aiohttp +import asyncio +from typing import List +from tqdm.asyncio import tqdm + +BASE_URL = "https://www.netztest.at/opendata" +SUBDOMAIN_DETAILS = "opentests" + + +def fetch_rtr_details(open_test_uuids: List[str]) -> List[dict]: + ''' + Fetch test details from RTR-Opendata for a list of open_test_uuids. + These open_test_uuids can for instance be obtained via data_extractor.fetch_rtr_overview() + + :param open_test_uuids: List of open_test_uuids for which test details will be fetched + + :return: A list of dictionaries with raw test results + ''' + + async def fetch(session, url): + async with session.get(url) as response: + return await response.json() + + async def query(urls): + tasks = [] + async with aiohttp.ClientSession() as session: + #Generate a task for each URL to fetch + tasks = [ + asyncio.create_task(fetch(session, url)) for url in urls + ] + + #Run tasks on the event loop and print progress via TQDM + return [ + await f for f in tqdm.as_completed(tasks) + ] + + urls = [f"{BASE_URL}/{SUBDOMAIN_DETAILS}/{uuid}" for uuid in open_test_uuids] + + loop = asyncio.get_event_loop() + results = loop.run_until_complete( + query(urls) + ) + + return results diff --git a/requirements.txt b/requirements.txt index a027b109661291f8de2b6f8114eba0fd347299fa..1301d8af5944e13d7e387ff0784e901323e45a02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ overpy>=0.4 geopandas>=0.8.1 cartopy>=0.18.0 aiohttp>=3.7.4 +tqdm>=4.60.0 diff --git a/tests/data_extractor_test.py b/tests/data_extractor_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f0c77696aa3f146b4323e46f4c137f6806decf26 --- /dev/null +++ b/tests/data_extractor_test.py @@ -0,0 +1,19 @@ +import unittest +import pandas as pd + +from context import measprocess as mpc + +class TestRTRExtractor(unittest.TestCase): + def setUp(self): + self.open_test_uuids = [ + 'Ob6c34648-54f3-435c-b5f9-5677c8694ad9', + 'Ocf041649-977f-4957-a27f-215069579c18' + ] + + def test_basics(self): + tests = mpc.data_extractor.fetch_rtr_details( + self.open_test_uuids + ) + self.assertTrue( + type(tests[0]) == dict + )