58 lines
1.6 KiB
Python
58 lines
1.6 KiB
Python
|
import pandas
|
||
|
|
|
||
|
|
import os
|
||
|
|
|
||
|
|
politicians = ['salvini', 'renzi']
|
||
|
|
valid_combination = set()
|
||
|
|
def create_dict(suffix, fread=None):
|
||
|
|
|
||
|
|
def read_file(filename):
|
||
|
|
if fread is None:
|
||
|
|
with open(filename, 'r') as f:
|
||
|
|
return f.read()
|
||
|
|
else:
|
||
|
|
return fread(filename)
|
||
|
|
|
||
|
|
obj = dict()
|
||
|
|
obj['all'] = dict()
|
||
|
|
for i in range(2013, 2020):
|
||
|
|
obj[i] = dict()
|
||
|
|
for p in politicians:
|
||
|
|
filename = f'dataset/{i}/{p}_{suffix}'
|
||
|
|
if os.path.exists(filename):
|
||
|
|
obj[i][p] = read_file(filename)
|
||
|
|
valid_combination.add(str(i)+':'+p)
|
||
|
|
|
||
|
|
# embed()
|
||
|
|
for p in politicians:
|
||
|
|
filename = f'dataset/all/{p}_{suffix}'
|
||
|
|
if os.path.exists(filename):
|
||
|
|
obj['all'][p] = read_file(filename)
|
||
|
|
valid_combination.add('all:'+p)
|
||
|
|
|
||
|
|
return obj
|
||
|
|
|
||
|
|
def load_jfile(fname):
|
||
|
|
import json
|
||
|
|
with open(fname, 'r') as f:
|
||
|
|
content = f.read()
|
||
|
|
return json.loads(content)
|
||
|
|
|
||
|
|
travelsHTML = create_dict('comuni.html')
|
||
|
|
counter = create_dict('counter', load_jfile)
|
||
|
|
# counter['all']['renzi'] = json.loads(counter['all']['renzi'])
|
||
|
|
# counter['all']['salvini'] = json.loads(counter['all']['salvini'])
|
||
|
|
emoji = create_dict('emoji', load_jfile)
|
||
|
|
words = create_dict('words')
|
||
|
|
sleep = create_dict('sleep', lambda fname: pandas.read_json(fname).drop('', axis=1).values.tolist())
|
||
|
|
tt = create_dict('trend.json', load_jfile)
|
||
|
|
|
||
|
|
# load trends tsv
|
||
|
|
import glob
|
||
|
|
tsv = dict()
|
||
|
|
l = len('dataset/trends/')
|
||
|
|
for g in glob.glob('dataset/trends/*.tsv'):
|
||
|
|
gname = g[l:]
|
||
|
|
with open(g, 'r') as f:
|
||
|
|
tsv[gname] = f.read()
|