Add location database ORM

Add API routes

app/__main__.py

@@ -0,0 +1,6 @@
+import uvicorn
+
+from app.app import app
+
+if __name__ == '__main__':
+    uvicorn.run(app)

app/api.py

@@ -0,0 +1,65 @@
+import sqlite3
+from typing import List
+
+from fastapi import APIRouter
+from fastapi.params import Depends
+
+from core import diyanetdb, diyanet
+from core.diyanet import PrayerTimes
+from core.diyanetdb import Location
+
+api = APIRouter()
+
+
+def get_connection():
+    with diyanetdb.get_connection() as conn:
+        yield conn
+
+
+@api.get('/diyanet/countries', response_model=List[str])
+async def list_countries(
+        conn: sqlite3.Connection = Depends(get_connection)
+):
+    return diyanetdb.find_countries(conn)
+
+
+@api.get('/diyanet/countries/{country}/cities', response_model=List[str])
+async def list_cities_in_country(
+        country: str,
+        conn: sqlite3.Connection = Depends(get_connection)
+):
+    cities = diyanetdb.find_cities(conn, country=country)
+    return cities
+
+
+@api.get('/diyanet/location/{country}', response_model=List[Location])
+async def list_locations_for_country(
+        country: str,
+        conn: sqlite3.Connection = Depends(get_connection)
+):
+    items = diyanetdb.find_locations(conn, country=country)
+    return items
+
+
+@api.get('/diyanet/location/{country}/{city}', response_model=List[Location])
+async def list_locations_for_city(
+        country: str,
+        city: str,
+        conn: sqlite3.Connection = Depends(get_connection)
+):
+    items = diyanetdb.find_locations(conn, country=country, city=city)
+    return items
+
+
+@api.get('/diyanet/prayertimes', response_model=List[PrayerTimes])
+async def get_prayer_times(location_id: int):
+    times = diyanet.get_prayer_times(location_id)
+    return times
+
+
+@api.get('/diyanet/search', response_model=List[PrayerTimes])
+async def search_location(
+        q: str,
+        conn: sqlite3.Connection = Depends(get_connection)):
+    locations = diyanetdb.find_location_by_name(conn, q)
+    return locations

app/app.py

@@ -0,0 +1,6 @@
+from fastapi import FastAPI
+
+from app.api import api as api_router
+
+app = FastAPI()
+app.include_router(api_router, prefix='/api')

app/config.py

@@ -0,0 +1,4 @@
+from os import getenv
+from pathlib import Path
+
+DB_PATH = Path(getenv('DB_PATH', Path(__file__).parent.parent / 'diyanetdb.sqlite3'))

core/diyanet.py

@@ -1,3 +1,4 @@
+import pprint
 from datetime import datetime
 from typing import List
 
@@ -32,10 +33,103 @@ async def parse_prayer_times(url: str) -> List[PrayerTimes]:
     return items
 
 
+def make_location_url(location_id: int) -> str:
+    return f'https://namazvakitleri.diyanet.gov.tr/en-US/{location_id}'
+
+
+async def get_prayer_times(location_id: int) -> List[PrayerTimes]:
+    url = make_location_url(location_id)
+    return await parse_prayer_times(url)
+
+
+async def fetch_locations() -> List[dict]:
+    countries = await _get_countries()
+
+    locations = []
+    for country_name, cities in countries.items():
+        cities: dict
+
+        country_id = cities['_countryId']
+        has_regions = cities['_hasRegions']
+        del cities['_countryId']
+        del cities['_hasRegions']
+
+        if not has_regions:
+            for (cname, cid) in cities.items():
+                locations.append(dict(
+                    country_id=country_id,
+                    country_name=country_name,
+                    city_id=cid,
+                    city_name=cname,
+                ))
+            continue
+
+        ctasks = (_get_regions(country_id, cid) for ckey, cid in cities.items())
+        regions = await asyncio.gather(*ctasks)
+
+        for (cname, cid), cregions in zip(cities.items(), regions):
+            for rname, rid in cregions.items():
+                locations.append(dict(
+                    country_id=country_id,
+                    country_name=country_name,
+                    city_id=cid,
+                    city_name=cname,
+                    region_id=rid,
+                    region_name=rname,
+                ))
+    return locations
+
+
+async def _get_cities(country_id: int) -> dict:
+    url = f'https://namazvakitleri.diyanet.gov.tr/en-US/home/GetRegList' \
+          f'?ChangeType=country&CountryId={country_id}&Culture=tr-TR'
+    res = await _http.get(url)
+    data = res.json()
+    if data['HasStateList']:
+        items = ((it['SehirAdi'], int(it['SehirID'])) for it in data['StateList'])
+    else:
+        items = ((it['IlceAdi'], int(it['IlceID'])) for it in data['StateRegionList'])
+    cities = dict(items)
+    cities['_countryId'] = country_id
+    cities['_hasRegions'] = data['HasStateList']
+    return cities
+
+
+async def _get_regions(country_id: int, city_id: int) -> dict:
+    url = f'https://namazvakitleri.diyanet.gov.tr/tr-TR/home/GetRegList' \
+          f'?ChangeType=state&CountryId={country_id}&Culture=tr-TR&StateId={city_id}'
+    res = await _http.get(url)
+    data = res.json()
+    items = ((it['IlceAdi'], int(it['IlceID'])) for it in data['StateRegionList'])
+    return dict(items)
+
+
+async def _get_countries() -> dict:
+    url = 'https://namazvakitleri.diyanet.gov.tr/tr-TR'
+    res = await _http.get(url)
+    soup = BeautifulSoup(res.text, 'html.parser')
+
+    countries = {}
+    for it in soup.select('select[name=country] option'):
+        countries[it.text] = int(it['value'])
+    return countries
+
+
 if __name__ == '__main__':
     import asyncio
-    import pprint
+    import core.diyanetdb
+
+
+    async def main():
+        with core.diyanetdb.get_connection() as conn:
+            id = core.diyanetdb.get_location_id(conn,
+                                                country_name='AVUSTURYA',
+                                                city_name='PESSENDELLACH')
+        url = make_location_url(id)
+        times = await parse_prayer_times(url)
+        print(times)
+
 
     loop = asyncio.get_event_loop()
-    results = loop.run_until_complete(parse_prayer_times('https://namazvakitleri.diyanet.gov.tr/en-US'))
+    results = loop.run_until_complete(main())
     pprint.pprint(results)

core/diyanetdb.py

@@ -0,0 +1,113 @@
+import sqlite3
+from contextlib import contextmanager
+from typing import List
+
+import httpx
+from pydantic import BaseModel
+
+from app import config
+
+_http = httpx.AsyncClient()
+
+
+class Location(BaseModel):
+    id: int
+    country: str
+    city: str
+    region: str = None
+
+
+@contextmanager
+def get_connection() -> sqlite3.Connection:
+    def dict_factory(cursor, row):
+        d = {}
+        for idx, col in enumerate(cursor.description):
+            d[col[0]] = row[idx]
+        return d
+
+    conn = sqlite3.connect(config.DB_PATH)
+    conn.row_factory = dict_factory
+    yield conn
+    conn.close()
+
+
+def _check_table_exists(conn: sqlite3.Connection) -> bool:
+    stmt = conn.execute("select count(*) as count from sqlite_master where type=? and name=?", ('table', 'locations'))
+    return bool(stmt.fetchone()['count'])
+
+
+def init_db(conn: sqlite3.Connection) -> bool:
+    if not _check_table_exists(conn):
+        conn.execute('''
+            CREATE TABLE IF NOT EXISTS locations (
+                id INTEGER UNIQUE NOT NULL PRIMARY KEY,
+                country TEXT NOT NULL,
+                city TEXT NOT NULL,
+                region TEXT
+            )
+        ''')
+    return _check_table_exists(conn)
+
+
+def save_locations(conn: sqlite3.Connection, locations: List[dict]):
+    sql = 'INSERT INTO locations (id, country, city, region) VALUES (:id, :country, :city, :region)'
+    conn.executemany(sql, locations)
+    conn.commit()
+
+
+def find_locations(conn: sqlite3.Connection,
+                   *,
+                   country: str,
+                   city: str = '') -> List[Location]:
+    sql = '''
+        SELECT DISTINCT id, country, city, region
+        FROM locations
+        {where}
+        ORDER BY city, region
+    '''
+    values = [country]
+    conditions = ['country = ?']
+    if city:
+        conditions.append('city = ?')
+        values.append(city)
+    sql = sql.format(where=f"WHERE {' and '.join(conditions)}")
+    stmt = conn.execute(sql, values)
+    rows = stmt.fetchall()
+    if not rows:
+        return []
+    return [Location(**row) for row in rows]
+
+
+def find_countries(conn: sqlite3.Connection) -> List[str]:
+    sql = 'select DISTINCT country from locations ORDER BY country'
+    stmt = conn.execute(sql)
+    rows = stmt.fetchall()
+    return [row['country'] for row in rows]
+
+
+def find_cities(conn: sqlite3.Connection, country: str) -> List[str]:
+    sql = 'select DISTINCT city from locations WHERE country = ? ORDER BY city'
+    stmt = conn.execute(sql, [country])
+    rows = stmt.fetchall()
+    return [row['city'] for row in rows]
+
+
+def find_location_by_name(conn: sqlite3.Connection, q: str) -> List[Location]:
+    sql = '''
+        SELECT DISTINCT id, country, city, region
+        FROM locations_search
+        WHERE locations_search match :q
+        ORDER BY country, city, region   
+    '''
+    stmt = conn.execute(sql, {'q': f'%{q}%'})
+    rows = stmt.fetchall()
+    if not rows:
+        return []
+    return [Location(**row) for row in rows]
+
+
+if __name__ == '__main__':
+    from pprint import pprint
+
+    with get_connection() as conn:
+        pprint(find_location_by_name(conn, 'ala'))