From bf81b4637fd72bf9f2910efbe67db2d439f25aa8 Mon Sep 17 00:00:00 2001
From: Lukas Eller <lukas.eller123@gmail.com>
Date: Wed, 21 Apr 2021 10:10:26 +0200
Subject: [PATCH] implemented fetch_rtr_details

---
 measprocess/__init__.py       |  2 ++
 measprocess/data_extractor.py | 44 +++++++++++++++++++++++++++++++++++
 requirements.txt              |  1 +
 tests/data_extractor_test.py  | 19 +++++++++++++++
 4 files changed, 66 insertions(+)
 create mode 100644 measprocess/data_extractor.py
 create mode 100644 tests/data_extractor_test.py

diff --git a/measprocess/__init__.py b/measprocess/__init__.py
index f619d50..04f65a1 100644
--- a/measprocess/__init__.py
+++ b/measprocess/__init__.py
@@ -4,3 +4,5 @@ from .geospatial import get_geoseries_streets, get_geoseries_blockages
 from .geospatial import project_onto_streets
 
 from .plotting import plot_series_osm
+
+from .data_extractor import fetch_rtr_details
diff --git a/measprocess/data_extractor.py b/measprocess/data_extractor.py
new file mode 100644
index 0000000..670b482
--- /dev/null
+++ b/measprocess/data_extractor.py
@@ -0,0 +1,44 @@
+import aiohttp
+import asyncio
+from typing import List
+from tqdm.asyncio import tqdm
+
+BASE_URL = "https://www.netztest.at/opendata"
+SUBDOMAIN_DETAILS = "opentests"
+
+
+def fetch_rtr_details(open_test_uuids: List[str]) -> List[dict]:
+    '''
+    Fetch test details from RTR-Opendata for a list of open_test_uuids.
+    These open_test_uuids can for instance be obtained via data_extractor.fetch_rtr_overview()
+
+    :param open_test_uuids: List of open_test_uuids for which test details will be fetched
+
+    :return: A list of dictionaries with raw test results
+    '''
+
+    async def fetch(session, url):
+        async with session.get(url) as response:
+            return await response.json()
+
+    async def query(urls):
+        tasks = []
+        async with aiohttp.ClientSession() as session:
+            #Generate a task for each URL to fetch
+            tasks = [
+                asyncio.create_task(fetch(session, url)) for url in urls
+            ]
+
+            #Run tasks on the event loop and print progress via TQDM
+            return [
+                await f for f in tqdm.as_completed(tasks)
+            ]
+
+    urls = [f"{BASE_URL}/{SUBDOMAIN_DETAILS}/{uuid}" for uuid in open_test_uuids]
+
+    loop = asyncio.get_event_loop()
+    results = loop.run_until_complete(
+        query(urls)
+    )
+
+    return results
diff --git a/requirements.txt b/requirements.txt
index a027b10..1301d8a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,3 +4,4 @@ overpy>=0.4
 geopandas>=0.8.1
 cartopy>=0.18.0
 aiohttp>=3.7.4
+tqdm>=4.60.0
diff --git a/tests/data_extractor_test.py b/tests/data_extractor_test.py
new file mode 100644
index 0000000..f0c7769
--- /dev/null
+++ b/tests/data_extractor_test.py
@@ -0,0 +1,19 @@
+import unittest
+import pandas as pd
+
+from context import measprocess as mpc
+
+class TestRTRExtractor(unittest.TestCase):
+    def setUp(self):
+        self.open_test_uuids = [
+            'Ob6c34648-54f3-435c-b5f9-5677c8694ad9',
+            'Ocf041649-977f-4957-a27f-215069579c18'
+        ]
+
+    def test_basics(self):
+        tests = mpc.data_extractor.fetch_rtr_details(
+            self.open_test_uuids
+        )
+        self.assertTrue(
+            type(tests[0]) == dict
+        )
-- 
2.22.0