From 8dbcb420b16d14a3e9c1c5ef8a17e207f1de6683 Mon Sep 17 00:00:00 2001 From: Abdussamet Kocak Date: Mon, 20 Jan 2020 19:05:38 +0300 Subject: [PATCH] Add location database ORM Add API routes --- app/__main__.py | 6 +++ app/api.py | 65 ++++++++++++++++++++++++++ app/app.py | 6 +++ app/config.py | 4 ++ core/diyanet.py | 98 +++++++++++++++++++++++++++++++++++++++- core/diyanetdb.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 290 insertions(+), 2 deletions(-) create mode 100644 app/__main__.py create mode 100644 app/api.py create mode 100644 app/config.py create mode 100644 core/diyanetdb.py diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 0000000..29ca97c --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,6 @@ +import uvicorn + +from app.app import app + +if __name__ == '__main__': + uvicorn.run(app) diff --git a/app/api.py b/app/api.py new file mode 100644 index 0000000..2cb6f32 --- /dev/null +++ b/app/api.py @@ -0,0 +1,65 @@ +import sqlite3 +from typing import List + +from fastapi import APIRouter +from fastapi.params import Depends + +from core import diyanetdb, diyanet +from core.diyanet import PrayerTimes +from core.diyanetdb import Location + +api = APIRouter() + + +def get_connection(): + with diyanetdb.get_connection() as conn: + yield conn + + +@api.get('/diyanet/countries', response_model=List[str]) +async def list_countries( + conn: sqlite3.Connection = Depends(get_connection) +): + return diyanetdb.find_countries(conn) + + +@api.get('/diyanet/countries/{country}/cities', response_model=List[str]) +async def list_cities_in_country( + country: str, + conn: sqlite3.Connection = Depends(get_connection) +): + cities = diyanetdb.find_cities(conn, country=country) + return cities + + +@api.get('/diyanet/location/{country}', response_model=List[Location]) +async def list_locations_for_country( + country: str, + conn: sqlite3.Connection = Depends(get_connection) +): + items = diyanetdb.find_locations(conn, country=country) + return items + + +@api.get('/diyanet/location/{country}/{city}', response_model=List[Location]) +async def list_locations_for_city( + country: str, + city: str, + conn: sqlite3.Connection = Depends(get_connection) +): + items = diyanetdb.find_locations(conn, country=country, city=city) + return items + + +@api.get('/diyanet/prayertimes', response_model=List[PrayerTimes]) +async def get_prayer_times(location_id: int): + times = diyanet.get_prayer_times(location_id) + return times + + +@api.get('/diyanet/search', response_model=List[PrayerTimes]) +async def search_location( + q: str, + conn: sqlite3.Connection = Depends(get_connection)): + locations = diyanetdb.find_location_by_name(conn, q) + return locations diff --git a/app/app.py b/app/app.py index e69de29..213ec5b 100644 --- a/app/app.py +++ b/app/app.py @@ -0,0 +1,6 @@ +from fastapi import FastAPI + +from app.api import api as api_router + +app = FastAPI() +app.include_router(api_router, prefix='/api') diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..c4545d2 --- /dev/null +++ b/app/config.py @@ -0,0 +1,4 @@ +from os import getenv +from pathlib import Path + +DB_PATH = Path(getenv('DB_PATH', Path(__file__).parent.parent / 'diyanetdb.sqlite3')) diff --git a/core/diyanet.py b/core/diyanet.py index 14d08c2..7e8768f 100644 --- a/core/diyanet.py +++ b/core/diyanet.py @@ -1,3 +1,4 @@ +import pprint from datetime import datetime from typing import List @@ -32,10 +33,103 @@ async def parse_prayer_times(url: str) -> List[PrayerTimes]: return items +def make_location_url(location_id: int) -> str: + return f'https://namazvakitleri.diyanet.gov.tr/en-US/{location_id}' + + +async def get_prayer_times(location_id: int) -> List[PrayerTimes]: + url = make_location_url(location_id) + return await parse_prayer_times(url) + + +async def fetch_locations() -> List[dict]: + countries = await _get_countries() + + locations = [] + for country_name, cities in countries.items(): + cities: dict + + country_id = cities['_countryId'] + has_regions = cities['_hasRegions'] + del cities['_countryId'] + del cities['_hasRegions'] + + if not has_regions: + for (cname, cid) in cities.items(): + locations.append(dict( + country_id=country_id, + country_name=country_name, + city_id=cid, + city_name=cname, + )) + continue + + ctasks = (_get_regions(country_id, cid) for ckey, cid in cities.items()) + regions = await asyncio.gather(*ctasks) + + for (cname, cid), cregions in zip(cities.items(), regions): + for rname, rid in cregions.items(): + locations.append(dict( + country_id=country_id, + country_name=country_name, + city_id=cid, + city_name=cname, + region_id=rid, + region_name=rname, + )) + return locations + + +async def _get_cities(country_id: int) -> dict: + url = f'https://namazvakitleri.diyanet.gov.tr/en-US/home/GetRegList' \ + f'?ChangeType=country&CountryId={country_id}&Culture=tr-TR' + res = await _http.get(url) + data = res.json() + if data['HasStateList']: + items = ((it['SehirAdi'], int(it['SehirID'])) for it in data['StateList']) + else: + items = ((it['IlceAdi'], int(it['IlceID'])) for it in data['StateRegionList']) + cities = dict(items) + cities['_countryId'] = country_id + cities['_hasRegions'] = data['HasStateList'] + return cities + + +async def _get_regions(country_id: int, city_id: int) -> dict: + url = f'https://namazvakitleri.diyanet.gov.tr/tr-TR/home/GetRegList' \ + f'?ChangeType=state&CountryId={country_id}&Culture=tr-TR&StateId={city_id}' + res = await _http.get(url) + data = res.json() + items = ((it['IlceAdi'], int(it['IlceID'])) for it in data['StateRegionList']) + return dict(items) + + +async def _get_countries() -> dict: + url = 'https://namazvakitleri.diyanet.gov.tr/tr-TR' + res = await _http.get(url) + soup = BeautifulSoup(res.text, 'html.parser') + + countries = {} + for it in soup.select('select[name=country] option'): + countries[it.text] = int(it['value']) + return countries + + if __name__ == '__main__': import asyncio - import pprint + import core.diyanetdb + + + async def main(): + with core.diyanetdb.get_connection() as conn: + id = core.diyanetdb.get_location_id(conn, + country_name='AVUSTURYA', + city_name='PESSENDELLACH') + url = make_location_url(id) + times = await parse_prayer_times(url) + print(times) + loop = asyncio.get_event_loop() - results = loop.run_until_complete(parse_prayer_times('https://namazvakitleri.diyanet.gov.tr/en-US')) + results = loop.run_until_complete(main()) pprint.pprint(results) diff --git a/core/diyanetdb.py b/core/diyanetdb.py new file mode 100644 index 0000000..e56ad1d --- /dev/null +++ b/core/diyanetdb.py @@ -0,0 +1,113 @@ +import sqlite3 +from contextlib import contextmanager +from typing import List + +import httpx +from pydantic import BaseModel + +from app import config + +_http = httpx.AsyncClient() + + +class Location(BaseModel): + id: int + country: str + city: str + region: str = None + + +@contextmanager +def get_connection() -> sqlite3.Connection: + def dict_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + + conn = sqlite3.connect(config.DB_PATH) + conn.row_factory = dict_factory + yield conn + conn.close() + + +def _check_table_exists(conn: sqlite3.Connection) -> bool: + stmt = conn.execute("select count(*) as count from sqlite_master where type=? and name=?", ('table', 'locations')) + return bool(stmt.fetchone()['count']) + + +def init_db(conn: sqlite3.Connection) -> bool: + if not _check_table_exists(conn): + conn.execute(''' + CREATE TABLE IF NOT EXISTS locations ( + id INTEGER UNIQUE NOT NULL PRIMARY KEY, + country TEXT NOT NULL, + city TEXT NOT NULL, + region TEXT + ) + ''') + return _check_table_exists(conn) + + +def save_locations(conn: sqlite3.Connection, locations: List[dict]): + sql = 'INSERT INTO locations (id, country, city, region) VALUES (:id, :country, :city, :region)' + conn.executemany(sql, locations) + conn.commit() + + +def find_locations(conn: sqlite3.Connection, + *, + country: str, + city: str = '') -> List[Location]: + sql = ''' + SELECT DISTINCT id, country, city, region + FROM locations + {where} + ORDER BY city, region + ''' + values = [country] + conditions = ['country = ?'] + if city: + conditions.append('city = ?') + values.append(city) + sql = sql.format(where=f"WHERE {' and '.join(conditions)}") + stmt = conn.execute(sql, values) + rows = stmt.fetchall() + if not rows: + return [] + return [Location(**row) for row in rows] + + +def find_countries(conn: sqlite3.Connection) -> List[str]: + sql = 'select DISTINCT country from locations ORDER BY country' + stmt = conn.execute(sql) + rows = stmt.fetchall() + return [row['country'] for row in rows] + + +def find_cities(conn: sqlite3.Connection, country: str) -> List[str]: + sql = 'select DISTINCT city from locations WHERE country = ? ORDER BY city' + stmt = conn.execute(sql, [country]) + rows = stmt.fetchall() + return [row['city'] for row in rows] + + +def find_location_by_name(conn: sqlite3.Connection, q: str) -> List[Location]: + sql = ''' + SELECT DISTINCT id, country, city, region + FROM locations_search + WHERE locations_search match :q + ORDER BY country, city, region + ''' + stmt = conn.execute(sql, {'q': f'%{q}%'}) + rows = stmt.fetchall() + if not rows: + return [] + return [Location(**row) for row in rows] + + +if __name__ == '__main__': + from pprint import pprint + + with get_connection() as conn: + pprint(find_location_by_name(conn, 'ala'))