import pandas

import os

politicians = ['salvini', 'renzi']
valid_combination = set()
def create_dict(suffix, fread=None):

    def read_file(filename):
        if fread is None:
            with open(filename, 'r') as f:
                return f.read()
        else:
            return fread(filename)

    obj = dict()
    obj['all'] = dict()
    for i in range(2013, 2020):
        obj[i] = dict()
        for p in politicians:
            filename = f'dataset/{i}/{p}_{suffix}'
            if os.path.exists(filename):
                obj[i][p] = read_file(filename)
                valid_combination.add(str(i)+':'+p)

    # embed()
    for p in politicians:
        filename = f'dataset/all/{p}_{suffix}'
        if os.path.exists(filename):
            obj['all'][p] = read_file(filename)
            valid_combination.add('all:'+p)

    return obj

def load_jfile(fname):
    import json
    with open(fname, 'r') as f:
        content = f.read()
    return json.loads(content)

travelsHTML = create_dict('comuni.html')
counter = create_dict('counter', load_jfile)
# counter['all']['renzi'] = json.loads(counter['all']['renzi'])
# counter['all']['salvini'] = json.loads(counter['all']['salvini'])
emoji = create_dict('emoji', load_jfile)
words = create_dict('words')
sleep = create_dict('sleep', lambda fname: pandas.read_json(fname).drop('', axis=1).values.tolist())
tt = create_dict('trend.json', load_jfile)

# load trends tsv
import glob
tsv = dict()
l = len('dataset/trends/')
for g in glob.glob('dataset/trends/*.tsv'):
    gname = g[l:]
    with open(g, 'r') as f:
        tsv[gname] = f.read()