thib8956/wallabag-tools
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: uses aio.gather to make parallel HTTP requests

Browse Source
This commit is contained in:
Thibaud Gasser 2020-03-13 00:30:18 +01:00
parent 94e57f51f5
commit 4f06791cad
1 changed files with 23 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
wallabag_mass_import.py
View File

@ -35,41 +35,42 @@ async def connect_to_wallabag(
) )
def read_urls_file(f): async def post_entry(w_api: Wallabag, item: Dict[str, str]):
while True: entry = await w_api.post_entries(
line = f.readline() url=item["url"],
if line: archive=item["is_archived"],
yield line starred=item["is_starred"],
else: original_url=item["origin_url"],
break tags=",".join(item["tags"]),
)
logging.info(f"Entry url {item['url']} posted to wallabag with id {entry['id']}")
return entry["id"]
async def delete_entry(w_api: Wallabag, item_id: int):
entry = await w_api.delete_entries(item_id)
logging.info(f"Deleted wallabag entry with id {entry['id']}")
async def import_articles(w_api: Wallabag, path: str, limit=0): async def import_articles(w_api: Wallabag, path: str, limit=0):
entries_id = [] entries_id = []
tasks = []
with open(path) as f: with open(path) as f:
for index, item in enumerate(json.load(f)): for index, item in enumerate(json.load(f)):
if limit > 0 and index == limit: if limit > 0 and index == limit:
break break
task = asyncio.ensure_future(post_entry(w_api, item))
tasks.append(task)
entry = await w_api.post_entries( entries_id = await asyncio.gather(*tasks)
url=item["url"],
archive=item["is_archived"],
starred=item["is_starred"],
original_url=item["origin_url"],
tags=",".join(item["tags"]),
)
logging.info(
f"Entry url {item['url']} posted to wallabag with id {entry['id']}"
)
entries_id.append(entry["id"])
return entries_id return entries_id
async def delete_all_entries(ids: List[int], w_api: Wallabag): async def delete_all_entries(ids: List[int], w_api: Wallabag):
tasks = []
for id in ids: for id in ids:
entry = await w_api.delete_entries(id) tasks.append(delete_entry(w_api, id))
logging.info(f"Deleted wallabag entry with id {entry['id']}") await asyncio.gather(*tasks)
async def main(): async def main():
@ -77,7 +78,7 @@ async def main():
configuration = dict(load_configuration(CONFIG_PATH)) configuration = dict(load_configuration(CONFIG_PATH))
w_api = await connect_to_wallabag(configuration, session) w_api = await connect_to_wallabag(configuration, session)
entries_id = await import_articles(w_api, "articles.json", limit=1) entries_id = await import_articles(w_api, ARTICLES_PATH, limit=4)
await delete_all_entries(entries_id, w_api) await delete_all_entries(entries_id, w_api)