eternal raven
from mysql import connector
from datetime import datetime
from typing import AsyncGenerator
import asyncio
async def get_data[T, U](query: str, times: int, chunk_size: int = 2000) -> AsyncGenerator[T, U]:
with connector.connect(host='localhost', user='root', password='*****', database='ffxivcensus') as connection:
start = datetime.now()
cursor = connection.cursor()
cursor.execute(query)
for i in range(times):
rows = cursor.fetchmany(size=chunk_size)
if not rows:
break
del rows
yield cursor.fetchmany(size=chunk_size)
print(f'Finished Yielding Query {datetime.now() - start}')
async def process() -> None:
start = datetime.now()
query = 'SELECT id FROM tblplayers WHERE race <> "N/A"'
async for j in get_data(query, 5):
print(f'Finished Getting Data {(datetime.now() - start)}')
print(f'Finished Everything {datetime.now() - start}')
if __name__ == '__main__':
asyncio.run(process())
Finished Getting Data 0:00:00.086670
Finished Getting Data 0:00:00.120042
Finished Getting Data 0:00:00.159855
Finished Getting Data 0:00:00.180179
Finished Getting Data 0:00:00.224756
Finished Yielding Query 0:00:00.187877
Finished Everything 0:00:58.940260
I don’t really understand why the Process function takes about a minute to finish despite the function it calls finishes yielding and the for loops finish in less than a second.
The database is over twenty million entries in the specific table I’m looking at but I’m only grabbing a very small amount of rows
modest yoke
wheat olive
frank pike
