Outlook Header Analysis

Downloads message headers from the MS Live POP3 mailbox into a local gdata store (~/.outlook_headers.gdbm), then analyses them. Only new messages (UIDLs not already in the store) are fetched on each run.

import sys, os, importlib
sys.path.insert(0, os.path.expanduser('~/py/popit3'))
import outlook_to_analysis as ota
importlib.reload(ota)

answer = input('Clear the header store? (yes/no): ').strip().lower()
mode = 'n' if answer == 'yes' else 'c'
with ota.open_store(mode=mode) as store:
    count = len(list(store.keys()))
if mode == 'n':
    print('Store cleared.')
else:
    print(f'Store has {count:,} messages.')

user, client_id, refresh_token = ota.load_credentials()
print(f'User:   {user}')
print(f'Store:  {ota.HEADER_STORE}')
with ota.open_store() as store:
    count = len(list(store.keys()))
print(f'Stored: {count:,} messages')

access_token, expires_in = ota.get_access_token(client_id, refresh_token)
print(f'Token obtained (expires in {expires_in}s)')

pop = ota.connect_pop3(user, access_token)
total, total_size = pop.stat()
print(f'POP3: {total:,} messages  {total_size/1_000_000:.1f} MB')

with ota.open_store() as store:
    last = None
    for fetched, skipped, grand_total in ota.fetch_new_headers(pop, store, progress_every=100):
        new_total = grand_total - skipped
        pct = 100 * fetched // max(new_total, 1)
        print(f'  {fetched:,}/{new_total:,} new fetched  ({skipped:,} already known)  {pct}%', end='\r')
        last = (fetched, skipped, grand_total)
    print()
    count = len(list(store.keys()))
if last:
    print(f'Fetched {last[0]:,} new messages. Store now has {count:,} total.')
else:
    print(f'Nothing new. Store has {count:,} messages.')

pop.quit()

with ota.open_store() as store:
    stats = ota.analyze_store(store)

total = stats['total']
lo, hi = stats['date_range']
print(f'Total messages: {total:,}')
print(f'Date range:     {lo[:16]}  to  {hi[:16]}')

col = 52
print(f'\n--- To: address ({len(stats["to_counts"])} unique) ---')
print(f'{"Address":<{col}}  {"Count":>6}  {"  %":>5}')
print('-' * (col + 16))
for addr, cnt in stats['to_counts'][:50]:
    print(f'{addr:<{col}}  {cnt:>6,}  {100*cnt/total:>5.1f}%')
if len(stats['to_counts']) > 50:
    print(f'  ... and {len(stats["to_counts"]) - 50} more')

print(f'\n--- Sender domain (top 20) ---')
print(f'{"Domain":<{col}}  {"Count":>6}')
print('-' * (col + 10))
for domain, cnt in stats['domain_counts'][:20]:
    print(f'{domain:<{col}}  {cnt:>6,}')

if stats['list_id_counts']:
    print(f'\n--- Mailing lists (List-Id, top 20) ---')
    print(f'{"List-Id":<{col}}  {"Count":>6}')
    print('-' * (col + 10))
    for lid, cnt in stats['list_id_counts'][:20]:
        print(f'{lid:<{col}}  {cnt:>6,}')