# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE
# -*- coding: utf-8 -*-
import dataiku
import json
import pandas as pd

answers = dataiku.Folder("jI8G2N4I")

# -------------------------------------------------------------------------------- NOTEBOOK-CELL: CODE
new_answers = {
    "timestamp": [],
    "version": [],
    "question": [],
    "answer": [],
    "feedback": []
}

answers_paths = answers.list_paths_in_partition()
if len(answers_paths) > 0:
    for path in answers_paths:
        with answers.get_download_stream(path) as f:
            data = json.loads(f.read())
            for k in ["timestamp", "version", "question", "answer", "feedback"]:
                new_answers[k].append(data[k])
        answers.delete_path(path)

    df = pd.DataFrame.from_dict(new_answers)
    df.astype({'version': 'str'}).dtypes

    dataiku.Dataset("past_answers").write_with_schema(df)